From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- rust/.gitignore | 10 + rust/.kunitconfig | 3 + rust/Makefile | 815 +++ rust/bindgen_parameters | 73 + rust/bindings/bindings_helper.h | 172 + rust/bindings/lib.rs | 85 + rust/build_error.rs | 31 + rust/compiler_builtins.rs | 107 + rust/exports.c | 29 + rust/ffi.rs | 50 + rust/helpers/atomic.c | 1035 +++ rust/helpers/atomic_ext.c | 87 + rust/helpers/auxiliary.c | 15 + rust/helpers/barrier.c | 48 + rust/helpers/binder.c | 27 + rust/helpers/bitmap.c | 10 + rust/helpers/bitops.c | 69 + rust/helpers/blk.c | 14 + rust/helpers/bug.c | 13 + rust/helpers/build_assert.c | 25 + rust/helpers/build_bug.c | 8 + rust/helpers/clk.c | 68 + rust/helpers/completion.c | 8 + rust/helpers/cpu.c | 8 + rust/helpers/cpufreq.c | 11 + rust/helpers/cpumask.c | 83 + rust/helpers/cred.c | 13 + rust/helpers/device.c | 32 + rust/helpers/dma-resv.c | 14 + rust/helpers/dma.c | 51 + rust/helpers/drm.c | 78 + rust/helpers/drm_gpuvm.c | 26 + rust/helpers/err.c | 18 + rust/helpers/fs.c | 12 + rust/helpers/fwctl.c | 17 + rust/helpers/gpu.c | 17 + rust/helpers/helpers.c | 108 + rust/helpers/interrupt.c | 13 + rust/helpers/io.c | 161 + rust/helpers/irq.c | 11 + rust/helpers/jump_label.c | 14 + rust/helpers/kunit.c | 8 + rust/helpers/list.c | 17 + rust/helpers/maple_tree.c | 9 + rust/helpers/mm.c | 50 + rust/helpers/mutex.c | 30 + rust/helpers/net/genetlink.c | 46 + rust/helpers/of.c | 8 + rust/helpers/page.c | 28 + rust/helpers/pci.c | 59 + rust/helpers/pid_namespace.c | 28 + rust/helpers/platform.c | 8 + rust/helpers/poll.c | 11 + rust/helpers/processor.c | 8 + rust/helpers/property.c | 8 + rust/helpers/pwm.c | 20 + rust/helpers/rbtree.c | 20 + rust/helpers/rcu.c | 13 + rust/helpers/refcount.c | 28 + rust/helpers/regulator.c | 57 + rust/helpers/scatterlist.c | 26 + rust/helpers/security.c | 48 + rust/helpers/serdev.c | 22 + rust/helpers/signal.c | 8 + rust/helpers/slab.c | 17 + rust/helpers/spinlock.c | 53 + rust/helpers/srcu.c | 35 + rust/helpers/string.c | 8 + rust/helpers/sync.c | 18 + rust/helpers/task.c | 57 + rust/helpers/time.c | 40 + rust/helpers/uaccess.c | 29 + rust/helpers/usb.c | 9 + rust/helpers/vmalloc.c | 16 + rust/helpers/wait.c | 8 + rust/helpers/workqueue.c | 16 + rust/helpers/xarray.c | 28 + rust/kernel/.gitignore | 5 + rust/kernel/Kconfig.test | 86 + rust/kernel/acpi.rs | 55 + rust/kernel/alloc.rs | 273 + rust/kernel/alloc/allocator.rs | 323 + rust/kernel/alloc/allocator/iter.rs | 100 + rust/kernel/alloc/kbox.rs | 758 ++ rust/kernel/alloc/kvec.rs | 1688 +++++ rust/kernel/alloc/kvec/errors.rs | 66 + rust/kernel/alloc/layout.rs | 121 + rust/kernel/auxiliary.rs | 587 ++ rust/kernel/bitfield.rs | 865 +++ rust/kernel/bitmap.rs | 620 ++ rust/kernel/bits.rs | 205 + rust/kernel/block.rs | 18 + rust/kernel/block/mq.rs | 103 + rust/kernel/block/mq/gen_disk.rs | 241 + rust/kernel/block/mq/operations.rs | 286 + rust/kernel/block/mq/request.rs | 257 + rust/kernel/block/mq/tag_set.rs | 85 + rust/kernel/bug.rs | 154 + rust/kernel/build_assert.rs | 219 + rust/kernel/clk.rs | 335 + rust/kernel/configfs.rs | 1047 +++ rust/kernel/cpu.rs | 152 + rust/kernel/cpufreq.rs | 1416 ++++ rust/kernel/cpumask.rs | 343 + rust/kernel/cred.rs | 92 + rust/kernel/debugfs.rs | 724 ++ rust/kernel/debugfs/callback_adapters.rs | 130 + rust/kernel/debugfs/entry.rs | 170 + rust/kernel/debugfs/file_ops.rs | 388 + rust/kernel/debugfs/traits.rs | 321 + rust/kernel/device.rs | 952 +++ rust/kernel/device/property.rs | 633 ++ rust/kernel/device_id.rs | 218 + rust/kernel/devres.rs | 557 ++ rust/kernel/dma.rs | 1251 ++++ rust/kernel/driver.rs | 326 + rust/kernel/drm/device.rs | 588 ++ rust/kernel/drm/driver.rs | 226 + rust/kernel/drm/file.rs | 103 + rust/kernel/drm/gem/mod.rs | 405 ++ rust/kernel/drm/gem/shmem.rs | 721 ++ rust/kernel/drm/gpuvm/mod.rs | 334 + rust/kernel/drm/gpuvm/sm_ops.rs | 429 ++ rust/kernel/drm/gpuvm/va.rs | 176 + rust/kernel/drm/gpuvm/vm_bo.rs | 258 + rust/kernel/drm/ioctl.rs | 217 + rust/kernel/drm/mod.rs | 26 + rust/kernel/error.rs | 663 ++ rust/kernel/faux.rs | 143 + rust/kernel/firmware.rs | 383 + rust/kernel/fmt.rs | 298 + rust/kernel/fs.rs | 11 + rust/kernel/fs/file.rs | 473 ++ rust/kernel/fs/kiocb.rs | 68 + rust/kernel/fwctl.rs | 593 ++ rust/kernel/generated_arch_reachable_asm.rs.S | 7 + rust/kernel/generated_arch_static_branch_asm.rs.S | 7 + rust/kernel/generated_arch_warn_asm.rs.S | 7 + rust/kernel/gpu.rs | 6 + rust/kernel/gpu/buddy.rs | 614 ++ rust/kernel/i2c.rs | 596 ++ rust/kernel/id_pool.rs | 295 + rust/kernel/impl_flags.rs | 275 + rust/kernel/init.rs | 284 + rust/kernel/interop.rs | 9 + rust/kernel/interop/list.rs | 339 + rust/kernel/interrupt.rs | 89 + rust/kernel/io.rs | 1754 +++++ rust/kernel/io/mem.rs | 329 + rust/kernel/io/poll.rs | 183 + rust/kernel/io/register.rs | 1027 +++ rust/kernel/io/resource.rs | 235 + rust/kernel/ioctl.rs | 72 + rust/kernel/iommu/mod.rs | 5 + rust/kernel/iommu/pgtable.rs | 264 + rust/kernel/iov.rs | 314 + rust/kernel/irq.rs | 24 + rust/kernel/irq/flags.rs | 126 + rust/kernel/irq/request.rs | 483 ++ rust/kernel/jump_label.rs | 86 + rust/kernel/kunit.rs | 354 + rust/kernel/lib.rs | 312 + rust/kernel/list.rs | 1222 ++++ rust/kernel/list/arc.rs | 500 ++ rust/kernel/list/arc_field.rs | 97 + rust/kernel/list/impl_list_item_mod.rs | 361 + rust/kernel/maple_tree.rs | 656 ++ rust/kernel/miscdevice.rs | 441 ++ rust/kernel/mm.rs | 297 + rust/kernel/mm/mmput_async.rs | 68 + rust/kernel/mm/virt.rs | 470 ++ rust/kernel/module.rs | 80 + rust/kernel/module_param.rs | 204 + rust/kernel/net/mod.rs | 8 + rust/kernel/net/netlink.rs | 337 + rust/kernel/net/phy.rs | 853 +++ rust/kernel/net/phy/reg.rs | 226 + rust/kernel/num.rs | 88 + rust/kernel/num/bounded.rs | 1165 +++ rust/kernel/num/casts.rs | 298 + rust/kernel/of.rs | 55 + rust/kernel/opp.rs | 1148 +++ rust/kernel/page.rs | 382 + rust/kernel/pci.rs | 545 ++ rust/kernel/pci/id.rs | 580 ++ rust/kernel/pci/io.rs | 348 + rust/kernel/pci/irq.rs | 236 + rust/kernel/pid_namespace.rs | 65 + rust/kernel/platform.rs | 589 ++ rust/kernel/prelude.rs | 126 + rust/kernel/print.rs | 578 ++ rust/kernel/processor.rs | 14 + rust/kernel/ptr.rs | 295 + rust/kernel/ptr/projection.rs | 366 + rust/kernel/pwm.rs | 739 ++ rust/kernel/rbtree.rs | 1433 ++++ rust/kernel/regulator.rs | 398 + rust/kernel/revocable.rs | 271 + rust/kernel/safety.rs | 53 + rust/kernel/scatterlist.rs | 492 ++ rust/kernel/security.rs | 111 + rust/kernel/seq_file.rs | 53 + rust/kernel/serdev.rs | 604 ++ rust/kernel/sizes.rs | 134 + rust/kernel/soc.rs | 135 + rust/kernel/std_vendor.rs | 164 + rust/kernel/str.rs | 918 +++ rust/kernel/str/parse_int.rs | 148 + rust/kernel/sync.rs | 171 + rust/kernel/sync/arc.rs | 918 +++ rust/kernel/sync/arc/std_vendor.rs | 30 + rust/kernel/sync/aref.rs | 249 + rust/kernel/sync/atomic.rs | 850 +++ rust/kernel/sync/atomic/internal.rs | 352 + rust/kernel/sync/atomic/ordering.rs | 104 + rust/kernel/sync/atomic/predefine.rs | 335 + rust/kernel/sync/barrier.rs | 132 + rust/kernel/sync/completion.rs | 114 + rust/kernel/sync/condvar.rs | 258 + rust/kernel/sync/lock.rs | 324 + rust/kernel/sync/lock/global.rs | 312 + rust/kernel/sync/lock/mutex.rs | 147 + rust/kernel/sync/lock/spinlock.rs | 465 ++ rust/kernel/sync/locked_by.rs | 169 + rust/kernel/sync/poll.rs | 179 + rust/kernel/sync/rcu.rs | 88 + rust/kernel/sync/refcount.rs | 116 + rust/kernel/sync/set_once.rs | 133 + rust/kernel/sync/srcu.rs | 171 + rust/kernel/task.rs | 435 ++ rust/kernel/time.rs | 582 ++ rust/kernel/time/delay.rs | 86 + rust/kernel/time/hrtimer.rs | 1120 +++ rust/kernel/time/hrtimer/arc.rs | 111 + rust/kernel/time/hrtimer/pin.rs | 115 + rust/kernel/time/hrtimer/pin_mut.rs | 118 + rust/kernel/time/hrtimer/tbox.rs | 131 + rust/kernel/tracepoint.rs | 49 + rust/kernel/transmute.rs | 229 + rust/kernel/types.rs | 453 ++ rust/kernel/types/for_lt.rs | 169 + rust/kernel/uaccess.rs | 664 ++ rust/kernel/usb.rs | 485 ++ rust/kernel/workqueue.rs | 1114 +++ rust/kernel/xarray.rs | 283 + rust/macros/concat_idents.rs | 36 + rust/macros/export.rs | 25 + rust/macros/fmt.rs | 96 + rust/macros/for_lt.rs | 265 + rust/macros/helpers.rs | 60 + rust/macros/kunit.rs | 172 + rust/macros/lib.rs | 539 ++ rust/macros/module.rs | 654 ++ rust/macros/paste.rs | 113 + rust/macros/vtable.rs | 136 + rust/pin-init/CONTRIBUTING.md | 72 + rust/pin-init/README.md | 247 + rust/pin-init/examples/big_struct_in_place.rs | 43 + rust/pin-init/examples/error.rs | 31 + rust/pin-init/examples/linked_list.rs | 168 + rust/pin-init/examples/mutex.rs | 217 + rust/pin-init/examples/pthread_mutex.rs | 183 + rust/pin-init/examples/static_init.rs | 121 + rust/pin-init/internal/src/diagnostics.rs | 44 + rust/pin-init/internal/src/init.rs | 482 ++ rust/pin-init/internal/src/lib.rs | 59 + rust/pin-init/internal/src/pin_data.rs | 537 ++ rust/pin-init/internal/src/pinned_drop.rs | 61 + rust/pin-init/internal/src/zeroable.rs | 78 + rust/pin-init/src/__internal.rs | 409 ++ rust/pin-init/src/alloc.rs | 160 + rust/pin-init/src/lib.rs | 1797 +++++ rust/proc-macro2/README.md | 13 + rust/proc-macro2/detection.rs | 77 + rust/proc-macro2/extra.rs | 153 + rust/proc-macro2/fallback.rs | 1258 ++++ rust/proc-macro2/lib.rs | 1355 ++++ rust/proc-macro2/location.rs | 31 + rust/proc-macro2/marker.rs | 19 + rust/proc-macro2/parse.rs | 997 +++ rust/proc-macro2/probe.rs | 12 + rust/proc-macro2/probe/proc_macro_span.rs | 53 + rust/proc-macro2/probe/proc_macro_span_file.rs | 16 + rust/proc-macro2/probe/proc_macro_span_location.rs | 23 + rust/proc-macro2/rcvec.rs | 148 + rust/proc-macro2/wrapper.rs | 986 +++ rust/quote/README.md | 12 + rust/quote/ext.rs | 112 + rust/quote/format.rs | 170 + rust/quote/ident_fragment.rs | 90 + rust/quote/lib.rs | 1456 ++++ rust/quote/runtime.rs | 494 ++ rust/quote/spanned.rs | 52 + rust/quote/to_tokens.rs | 273 + rust/syn/README.md | 13 + rust/syn/attr.rs | 838 +++ rust/syn/bigint.rs | 68 + rust/syn/buffer.rs | 436 ++ rust/syn/classify.rs | 313 + rust/syn/custom_keyword.rs | 262 + rust/syn/custom_punctuation.rs | 306 + rust/syn/data.rs | 426 ++ rust/syn/derive.rs | 261 + rust/syn/discouraged.rs | 227 + rust/syn/drops.rs | 60 + rust/syn/error.rs | 469 ++ rust/syn/export.rs | 75 + rust/syn/expr.rs | 4175 +++++++++++ rust/syn/ext.rs | 138 + rust/syn/file.rs | 127 + rust/syn/fixup.rs | 775 ++ rust/syn/gen/clone.rs | 2269 ++++++ rust/syn/gen/debug.rs | 3240 +++++++++ rust/syn/gen/eq.rs | 2308 ++++++ rust/syn/gen/fold.rs | 3904 ++++++++++ rust/syn/gen/hash.rs | 2878 ++++++++ rust/syn/gen/visit.rs | 3943 ++++++++++ rust/syn/gen/visit_mut.rs | 3761 ++++++++++ rust/syn/generics.rs | 1479 ++++ rust/syn/group.rs | 293 + rust/syn/ident.rs | 110 + rust/syn/item.rs | 3492 +++++++++ rust/syn/lib.rs | 1013 +++ rust/syn/lifetime.rs | 158 + rust/syn/lit.rs | 1862 +++++ rust/syn/lookahead.rs | 334 + rust/syn/mac.rs | 227 + rust/syn/macros.rs | 184 + rust/syn/meta.rs | 429 ++ rust/syn/op.rs | 221 + rust/syn/parse.rs | 1421 ++++ rust/syn/parse_macro_input.rs | 130 + rust/syn/parse_quote.rs | 242 + rust/syn/pat.rs | 957 +++ rust/syn/path.rs | 968 +++ rust/syn/precedence.rs | 212 + rust/syn/print.rs | 18 + rust/syn/punctuated.rs | 1157 +++ rust/syn/restriction.rs | 180 + rust/syn/scan_expr.rs | 267 + rust/syn/sealed.rs | 6 + rust/syn/span.rs | 65 + rust/syn/spanned.rs | 120 + rust/syn/stmt.rs | 486 ++ rust/syn/thread.rs | 62 + rust/syn/token.rs | 1098 +++ rust/syn/tt.rs | 109 + rust/syn/ty.rs | 1273 ++++ rust/syn/verbatim.rs | 35 + rust/syn/whitespace.rs | 67 + rust/uapi/lib.rs | 42 + rust/uapi/uapi_helper.h | 17 + rust/zerocopy-derive/README.md | 14 + rust/zerocopy-derive/derive/from_bytes.rs | 192 + rust/zerocopy-derive/derive/into_bytes.rs | 164 + rust/zerocopy-derive/derive/known_layout.rs | 349 + rust/zerocopy-derive/derive/mod.rs | 136 + rust/zerocopy-derive/derive/try_from_bytes.rs | 765 ++ rust/zerocopy-derive/derive/unaligned.rs | 80 + rust/zerocopy-derive/lib.rs | 180 + rust/zerocopy-derive/repr.rs | 851 +++ rust/zerocopy-derive/util.rs | 863 +++ rust/zerocopy/README.md | 13 + rust/zerocopy/benches/as_bytes_dynamic_size.rs | 9 + rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 | 5 + .../benches/as_bytes_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/as_bytes_static_size.rs | 9 + rust/zerocopy/benches/as_bytes_static_size.x86-64 | 4 + .../benches/as_bytes_static_size.x86-64.mca | 45 + rust/zerocopy/benches/extend_vec_zeroed.rs | 9 + rust/zerocopy/benches/extend_vec_zeroed.x86-64 | 60 + rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca | 147 + .../benches/formats/coco_dynamic_padding.rs | 24 + rust/zerocopy/benches/formats/coco_dynamic_size.rs | 27 + rust/zerocopy/benches/formats/coco_static_size.rs | 27 + rust/zerocopy/benches/insert_vec_zeroed.rs | 13 + rust/zerocopy/benches/insert_vec_zeroed.x86-64 | 79 + rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca | 183 + rust/zerocopy/benches/new_box_zeroed.rs | 9 + rust/zerocopy/benches/new_box_zeroed.x86-64 | 7 + rust/zerocopy/benches/new_box_zeroed.x86-64.mca | 51 + .../new_box_zeroed_with_elems_dynamic_padding.rs | 11 + ...ew_box_zeroed_with_elems_dynamic_padding.x86-64 | 24 + ...ox_zeroed_with_elems_dynamic_padding.x86-64.mca | 81 + .../new_box_zeroed_with_elems_dynamic_size.rs | 9 + .../new_box_zeroed_with_elems_dynamic_size.x86-64 | 22 + ...w_box_zeroed_with_elems_dynamic_size.x86-64.mca | 77 + rust/zerocopy/benches/new_vec_zeroed.rs | 9 + rust/zerocopy/benches/new_vec_zeroed.x86-64 | 40 + rust/zerocopy/benches/new_vec_zeroed.x86-64.mca | 113 + rust/zerocopy/benches/new_zeroed.rs | 9 + rust/zerocopy/benches/new_zeroed.x86-64 | 3 + rust/zerocopy/benches/new_zeroed.x86-64.mca | 43 + rust/zerocopy/benches/read_from_bytes.rs | 7 + rust/zerocopy/benches/read_from_bytes.x86-64 | 15 + rust/zerocopy/benches/read_from_bytes.x86-64.mca | 65 + rust/zerocopy/benches/read_from_prefix.rs | 10 + rust/zerocopy/benches/read_from_prefix.x86-64 | 14 + rust/zerocopy/benches/read_from_prefix.x86-64.mca | 63 + rust/zerocopy/benches/read_from_suffix.rs | 10 + rust/zerocopy/benches/read_from_suffix.x86-64 | 15 + rust/zerocopy/benches/read_from_suffix.x86-64.mca | 65 + .../benches/ref_from_bytes_dynamic_padding.rs | 7 + .../benches/ref_from_bytes_dynamic_padding.x86-64 | 22 + .../ref_from_bytes_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_bytes_dynamic_size.rs | 7 + .../benches/ref_from_bytes_dynamic_size.x86-64 | 20 + .../benches/ref_from_bytes_dynamic_size.x86-64.mca | 75 + .../zerocopy/benches/ref_from_bytes_static_size.rs | 7 + .../benches/ref_from_bytes_static_size.x86-64 | 8 + .../benches/ref_from_bytes_static_size.x86-64.mca | 53 + .../ref_from_bytes_with_elems_dynamic_padding.rs | 10 + ...ef_from_bytes_with_elems_dynamic_padding.x86-64 | 19 + ...rom_bytes_with_elems_dynamic_padding.x86-64.mca | 71 + .../ref_from_bytes_with_elems_dynamic_size.rs | 10 + .../ref_from_bytes_with_elems_dynamic_size.x86-64 | 16 + ...f_from_bytes_with_elems_dynamic_size.x86-64.mca | 65 + .../benches/ref_from_prefix_dynamic_padding.rs | 10 + .../benches/ref_from_prefix_dynamic_padding.x86-64 | 22 + .../ref_from_prefix_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_prefix_dynamic_size.rs | 10 + .../benches/ref_from_prefix_dynamic_size.x86-64 | 17 + .../ref_from_prefix_dynamic_size.x86-64.mca | 67 + .../benches/ref_from_prefix_static_size.rs | 10 + .../benches/ref_from_prefix_static_size.x86-64 | 8 + .../benches/ref_from_prefix_static_size.x86-64.mca | 53 + .../ref_from_prefix_with_elems_dynamic_padding.rs | 13 + ...f_from_prefix_with_elems_dynamic_padding.x86-64 | 26 + ...om_prefix_with_elems_dynamic_padding.x86-64.mca | 85 + .../ref_from_prefix_with_elems_dynamic_size.rs | 13 + .../ref_from_prefix_with_elems_dynamic_size.x86-64 | 22 + ..._from_prefix_with_elems_dynamic_size.x86-64.mca | 77 + .../benches/ref_from_suffix_dynamic_padding.rs | 10 + .../benches/ref_from_suffix_dynamic_padding.x86-64 | 23 + .../ref_from_suffix_dynamic_padding.x86-64.mca | 79 + .../benches/ref_from_suffix_dynamic_size.rs | 10 + .../benches/ref_from_suffix_dynamic_size.x86-64 | 13 + .../ref_from_suffix_dynamic_size.x86-64.mca | 63 + .../benches/ref_from_suffix_static_size.rs | 10 + .../benches/ref_from_suffix_static_size.x86-64 | 13 + .../benches/ref_from_suffix_static_size.x86-64.mca | 61 + .../ref_from_suffix_with_elems_dynamic_padding.rs | 13 + ...f_from_suffix_with_elems_dynamic_padding.x86-64 | 27 + ...om_suffix_with_elems_dynamic_padding.x86-64.mca | 85 + .../ref_from_suffix_with_elems_dynamic_size.rs | 13 + .../ref_from_suffix_with_elems_dynamic_size.x86-64 | 23 + ..._from_suffix_with_elems_dynamic_size.x86-64.mca | 77 + rust/zerocopy/benches/split_at_dynamic_padding.rs | 12 + .../benches/split_at_dynamic_padding.x86-64 | 12 + .../benches/split_at_dynamic_padding.x86-64.mca | 59 + rust/zerocopy/benches/split_at_dynamic_size.rs | 12 + rust/zerocopy/benches/split_at_dynamic_size.x86-64 | 12 + .../benches/split_at_dynamic_size.x86-64.mca | 59 + .../benches/split_at_unchecked_dynamic_padding.rs | 12 + .../split_at_unchecked_dynamic_padding.x86-64 | 6 + .../split_at_unchecked_dynamic_padding.x86-64.mca | 49 + .../benches/split_at_unchecked_dynamic_size.rs | 12 + .../benches/split_at_unchecked_dynamic_size.x86-64 | 6 + .../split_at_unchecked_dynamic_size.x86-64.mca | 49 + .../benches/split_via_immutable_dynamic_padding.rs | 11 + .../split_via_immutable_dynamic_padding.x86-64 | 14 + .../split_via_immutable_dynamic_padding.x86-64.mca | 65 + .../benches/split_via_immutable_dynamic_size.rs | 11 + .../split_via_immutable_dynamic_size.x86-64 | 13 + .../split_via_immutable_dynamic_size.x86-64.mca | 63 + .../split_via_runtime_check_dynamic_padding.rs | 11 + .../split_via_runtime_check_dynamic_padding.x86-64 | 22 + ...it_via_runtime_check_dynamic_padding.x86-64.mca | 79 + .../split_via_runtime_check_dynamic_size.rs | 11 + .../split_via_runtime_check_dynamic_size.x86-64 | 13 + ...split_via_runtime_check_dynamic_size.x86-64.mca | 63 + .../benches/split_via_unchecked_dynamic_padding.rs | 11 + .../split_via_unchecked_dynamic_padding.x86-64 | 14 + .../split_via_unchecked_dynamic_padding.x86-64.mca | 65 + .../benches/split_via_unchecked_dynamic_size.rs | 11 + .../split_via_unchecked_dynamic_size.x86-64 | 13 + .../split_via_unchecked_dynamic_size.x86-64.mca | 63 + rust/zerocopy/benches/transmute.rs | 16 + rust/zerocopy/benches/transmute.x86-64 | 3 + rust/zerocopy/benches/transmute.x86-64.mca | 43 + .../zerocopy/benches/transmute_ref_dynamic_size.rs | 16 + .../benches/transmute_ref_dynamic_size.x86-64 | 4 + .../benches/transmute_ref_dynamic_size.x86-64.mca | 45 + rust/zerocopy/benches/transmute_ref_static_size.rs | 15 + .../benches/transmute_ref_static_size.x86-64 | 3 + .../benches/transmute_ref_static_size.x86-64.mca | 43 + rust/zerocopy/benches/try_read_from_bytes.rs | 7 + rust/zerocopy/benches/try_read_from_bytes.x86-64 | 23 + .../benches/try_read_from_bytes.x86-64.mca | 79 + rust/zerocopy/benches/try_read_from_prefix.rs | 10 + rust/zerocopy/benches/try_read_from_prefix.x86-64 | 16 + .../benches/try_read_from_prefix.x86-64.mca | 67 + rust/zerocopy/benches/try_read_from_suffix.rs | 10 + rust/zerocopy/benches/try_read_from_suffix.x86-64 | 18 + .../benches/try_read_from_suffix.x86-64.mca | 71 + .../benches/try_ref_from_bytes_dynamic_padding.rs | 7 + .../try_ref_from_bytes_dynamic_padding.x86-64 | 24 + .../try_ref_from_bytes_dynamic_padding.x86-64.mca | 81 + .../benches/try_ref_from_bytes_dynamic_size.rs | 7 + .../benches/try_ref_from_bytes_dynamic_size.x86-64 | 22 + .../try_ref_from_bytes_dynamic_size.x86-64.mca | 79 + .../benches/try_ref_from_bytes_static_size.rs | 7 + .../benches/try_ref_from_bytes_static_size.x86-64 | 13 + .../try_ref_from_bytes_static_size.x86-64.mca | 59 + ...ry_ref_from_bytes_with_elems_dynamic_padding.rs | 10 + ...ef_from_bytes_with_elems_dynamic_padding.x86-64 | 21 + ...rom_bytes_with_elems_dynamic_padding.x86-64.mca | 75 + .../try_ref_from_bytes_with_elems_dynamic_size.rs | 10 + ...y_ref_from_bytes_with_elems_dynamic_size.x86-64 | 18 + ...f_from_bytes_with_elems_dynamic_size.x86-64.mca | 69 + .../benches/try_ref_from_prefix_dynamic_padding.rs | 10 + .../try_ref_from_prefix_dynamic_padding.x86-64 | 29 + .../try_ref_from_prefix_dynamic_padding.x86-64.mca | 91 + .../benches/try_ref_from_prefix_dynamic_size.rs | 10 + .../try_ref_from_prefix_dynamic_size.x86-64 | 22 + .../try_ref_from_prefix_dynamic_size.x86-64.mca | 77 + .../benches/try_ref_from_prefix_static_size.rs | 10 + .../benches/try_ref_from_prefix_static_size.x86-64 | 15 + .../try_ref_from_prefix_static_size.x86-64.mca | 63 + ...y_ref_from_prefix_with_elems_dynamic_padding.rs | 13 + ...f_from_prefix_with_elems_dynamic_padding.x86-64 | 30 + ...om_prefix_with_elems_dynamic_padding.x86-64.mca | 91 + .../try_ref_from_prefix_with_elems_dynamic_size.rs | 13 + ..._ref_from_prefix_with_elems_dynamic_size.x86-64 | 26 + ..._from_prefix_with_elems_dynamic_size.x86-64.mca | 83 + .../benches/try_ref_from_suffix_dynamic_padding.rs | 10 + .../try_ref_from_suffix_dynamic_padding.x86-64 | 26 + .../try_ref_from_suffix_dynamic_padding.x86-64.mca | 85 + .../benches/try_ref_from_suffix_dynamic_size.rs | 10 + .../try_ref_from_suffix_dynamic_size.x86-64 | 18 + .../try_ref_from_suffix_dynamic_size.x86-64.mca | 71 + .../benches/try_ref_from_suffix_static_size.rs | 10 + .../benches/try_ref_from_suffix_static_size.x86-64 | 16 + .../try_ref_from_suffix_static_size.x86-64.mca | 67 + ...y_ref_from_suffix_with_elems_dynamic_padding.rs | 13 + ...f_from_suffix_with_elems_dynamic_padding.x86-64 | 32 + ...om_suffix_with_elems_dynamic_padding.x86-64.mca | 95 + .../try_ref_from_suffix_with_elems_dynamic_size.rs | 13 + ..._ref_from_suffix_with_elems_dynamic_size.x86-64 | 28 + ..._from_suffix_with_elems_dynamic_size.x86-64.mca | 87 + rust/zerocopy/benches/try_transmute.rs | 16 + rust/zerocopy/benches/try_transmute.x86-64 | 9 + rust/zerocopy/benches/try_transmute.x86-64.mca | 55 + .../benches/try_transmute_ref_dynamic_size.rs | 18 + .../benches/try_transmute_ref_dynamic_size.x86-64 | 6 + .../try_transmute_ref_dynamic_size.x86-64.mca | 49 + .../benches/try_transmute_ref_static_size.rs | 17 + .../benches/try_transmute_ref_static_size.x86-64 | 5 + .../try_transmute_ref_static_size.x86-64.mca | 47 + rust/zerocopy/benches/write_to_dynamic_size.rs | 9 + rust/zerocopy/benches/write_to_dynamic_size.x86-64 | 21 + .../benches/write_to_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_dynamic_size.rs | 12 + .../benches/write_to_prefix_dynamic_size.x86-64 | 21 + .../write_to_prefix_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_static_size.rs | 12 + .../benches/write_to_prefix_static_size.x86-64 | 11 + .../benches/write_to_prefix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/write_to_static_size.rs | 9 + rust/zerocopy/benches/write_to_static_size.x86-64 | 11 + .../benches/write_to_static_size.x86-64.mca | 57 + .../benches/write_to_suffix_dynamic_size.rs | 12 + .../benches/write_to_suffix_dynamic_size.x86-64 | 22 + .../write_to_suffix_dynamic_size.x86-64.mca | 79 + .../benches/write_to_suffix_static_size.rs | 12 + .../benches/write_to_suffix_static_size.x86-64 | 11 + .../benches/write_to_suffix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/zero_dynamic_padding.rs | 9 + rust/zerocopy/benches/zero_dynamic_padding.x86-64 | 7 + .../benches/zero_dynamic_padding.x86-64.mca | 51 + rust/zerocopy/benches/zero_dynamic_size.rs | 9 + rust/zerocopy/benches/zero_dynamic_size.x86-64 | 5 + rust/zerocopy/benches/zero_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/zero_static_size.rs | 9 + rust/zerocopy/benches/zero_static_size.x86-64 | 4 + rust/zerocopy/benches/zero_static_size.x86-64.mca | 45 + rust/zerocopy/rustdoc/style.css | 56 + rust/zerocopy/src/byte_slice.rs | 434 ++ rust/zerocopy/src/byteorder.rs | 1596 ++++ rust/zerocopy/src/deprecated.rs | 281 + rust/zerocopy/src/error.rs | 1350 ++++ rust/zerocopy/src/impls.rs | 2389 ++++++ rust/zerocopy/src/layout.rs | 2227 ++++++ rust/zerocopy/src/lib.rs | 7614 ++++++++++++++++++++ rust/zerocopy/src/macros.rs | 1825 +++++ rust/zerocopy/src/pointer/inner.rs | 754 ++ rust/zerocopy/src/pointer/invariant.rs | 298 + rust/zerocopy/src/pointer/mod.rs | 410 ++ rust/zerocopy/src/pointer/ptr.rs | 1586 ++++ rust/zerocopy/src/pointer/transmute.rs | 522 ++ rust/zerocopy/src/ref.rs | 1358 ++++ rust/zerocopy/src/split_at.rs | 1090 +++ rust/zerocopy/src/util/macro_util.rs | 1310 ++++ rust/zerocopy/src/util/macros.rs | 1067 +++ rust/zerocopy/src/util/mod.rs | 944 +++ rust/zerocopy/src/wrappers.rs | 1034 +++ 596 files changed, 168179 insertions(+) create mode 100644 rust/.gitignore create mode 100644 rust/.kunitconfig create mode 100644 rust/Makefile create mode 100644 rust/bindgen_parameters create mode 100644 rust/bindings/bindings_helper.h create mode 100644 rust/bindings/lib.rs create mode 100644 rust/build_error.rs create mode 100644 rust/compiler_builtins.rs create mode 100644 rust/exports.c create mode 100644 rust/ffi.rs create mode 100644 rust/helpers/atomic.c create mode 100644 rust/helpers/atomic_ext.c create mode 100644 rust/helpers/auxiliary.c create mode 100644 rust/helpers/barrier.c create mode 100644 rust/helpers/binder.c create mode 100644 rust/helpers/bitmap.c create mode 100644 rust/helpers/bitops.c create mode 100644 rust/helpers/blk.c create mode 100644 rust/helpers/bug.c create mode 100644 rust/helpers/build_assert.c create mode 100644 rust/helpers/build_bug.c create mode 100644 rust/helpers/clk.c create mode 100644 rust/helpers/completion.c create mode 100644 rust/helpers/cpu.c create mode 100644 rust/helpers/cpufreq.c create mode 100644 rust/helpers/cpumask.c create mode 100644 rust/helpers/cred.c create mode 100644 rust/helpers/device.c create mode 100644 rust/helpers/dma-resv.c create mode 100644 rust/helpers/dma.c create mode 100644 rust/helpers/drm.c create mode 100644 rust/helpers/drm_gpuvm.c create mode 100644 rust/helpers/err.c create mode 100644 rust/helpers/fs.c create mode 100644 rust/helpers/fwctl.c create mode 100644 rust/helpers/gpu.c create mode 100644 rust/helpers/helpers.c create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/helpers/io.c create mode 100644 rust/helpers/irq.c create mode 100644 rust/helpers/jump_label.c create mode 100644 rust/helpers/kunit.c create mode 100644 rust/helpers/list.c create mode 100644 rust/helpers/maple_tree.c create mode 100644 rust/helpers/mm.c create mode 100644 rust/helpers/mutex.c create mode 100644 rust/helpers/net/genetlink.c create mode 100644 rust/helpers/of.c create mode 100644 rust/helpers/page.c create mode 100644 rust/helpers/pci.c create mode 100644 rust/helpers/pid_namespace.c create mode 100644 rust/helpers/platform.c create mode 100644 rust/helpers/poll.c create mode 100644 rust/helpers/processor.c create mode 100644 rust/helpers/property.c create mode 100644 rust/helpers/pwm.c create mode 100644 rust/helpers/rbtree.c create mode 100644 rust/helpers/rcu.c create mode 100644 rust/helpers/refcount.c create mode 100644 rust/helpers/regulator.c create mode 100644 rust/helpers/scatterlist.c create mode 100644 rust/helpers/security.c create mode 100644 rust/helpers/serdev.c create mode 100644 rust/helpers/signal.c create mode 100644 rust/helpers/slab.c create mode 100644 rust/helpers/spinlock.c create mode 100644 rust/helpers/srcu.c create mode 100644 rust/helpers/string.c create mode 100644 rust/helpers/sync.c create mode 100644 rust/helpers/task.c create mode 100644 rust/helpers/time.c create mode 100644 rust/helpers/uaccess.c create mode 100644 rust/helpers/usb.c create mode 100644 rust/helpers/vmalloc.c create mode 100644 rust/helpers/wait.c create mode 100644 rust/helpers/workqueue.c create mode 100644 rust/helpers/xarray.c create mode 100644 rust/kernel/.gitignore create mode 100644 rust/kernel/Kconfig.test create mode 100644 rust/kernel/acpi.rs create mode 100644 rust/kernel/alloc.rs create mode 100644 rust/kernel/alloc/allocator.rs create mode 100644 rust/kernel/alloc/allocator/iter.rs create mode 100644 rust/kernel/alloc/kbox.rs create mode 100644 rust/kernel/alloc/kvec.rs create mode 100644 rust/kernel/alloc/kvec/errors.rs create mode 100644 rust/kernel/alloc/layout.rs create mode 100644 rust/kernel/auxiliary.rs create mode 100644 rust/kernel/bitfield.rs create mode 100644 rust/kernel/bitmap.rs create mode 100644 rust/kernel/bits.rs create mode 100644 rust/kernel/block.rs create mode 100644 rust/kernel/block/mq.rs create mode 100644 rust/kernel/block/mq/gen_disk.rs create mode 100644 rust/kernel/block/mq/operations.rs create mode 100644 rust/kernel/block/mq/request.rs create mode 100644 rust/kernel/block/mq/tag_set.rs create mode 100644 rust/kernel/bug.rs create mode 100644 rust/kernel/build_assert.rs create mode 100644 rust/kernel/clk.rs create mode 100644 rust/kernel/configfs.rs create mode 100644 rust/kernel/cpu.rs create mode 100644 rust/kernel/cpufreq.rs create mode 100644 rust/kernel/cpumask.rs create mode 100644 rust/kernel/cred.rs create mode 100644 rust/kernel/debugfs.rs create mode 100644 rust/kernel/debugfs/callback_adapters.rs create mode 100644 rust/kernel/debugfs/entry.rs create mode 100644 rust/kernel/debugfs/file_ops.rs create mode 100644 rust/kernel/debugfs/traits.rs create mode 100644 rust/kernel/device.rs create mode 100644 rust/kernel/device/property.rs create mode 100644 rust/kernel/device_id.rs create mode 100644 rust/kernel/devres.rs create mode 100644 rust/kernel/dma.rs create mode 100644 rust/kernel/driver.rs create mode 100644 rust/kernel/drm/device.rs create mode 100644 rust/kernel/drm/driver.rs create mode 100644 rust/kernel/drm/file.rs create mode 100644 rust/kernel/drm/gem/mod.rs create mode 100644 rust/kernel/drm/gem/shmem.rs create mode 100644 rust/kernel/drm/gpuvm/mod.rs create mode 100644 rust/kernel/drm/gpuvm/sm_ops.rs create mode 100644 rust/kernel/drm/gpuvm/va.rs create mode 100644 rust/kernel/drm/gpuvm/vm_bo.rs create mode 100644 rust/kernel/drm/ioctl.rs create mode 100644 rust/kernel/drm/mod.rs create mode 100644 rust/kernel/error.rs create mode 100644 rust/kernel/faux.rs create mode 100644 rust/kernel/firmware.rs create mode 100644 rust/kernel/fmt.rs create mode 100644 rust/kernel/fs.rs create mode 100644 rust/kernel/fs/file.rs create mode 100644 rust/kernel/fs/kiocb.rs create mode 100644 rust/kernel/fwctl.rs create mode 100644 rust/kernel/generated_arch_reachable_asm.rs.S create mode 100644 rust/kernel/generated_arch_static_branch_asm.rs.S create mode 100644 rust/kernel/generated_arch_warn_asm.rs.S create mode 100644 rust/kernel/gpu.rs create mode 100644 rust/kernel/gpu/buddy.rs create mode 100644 rust/kernel/i2c.rs create mode 100644 rust/kernel/id_pool.rs create mode 100644 rust/kernel/impl_flags.rs create mode 100644 rust/kernel/init.rs create mode 100644 rust/kernel/interop.rs create mode 100644 rust/kernel/interop/list.rs create mode 100644 rust/kernel/interrupt.rs create mode 100644 rust/kernel/io.rs create mode 100644 rust/kernel/io/mem.rs create mode 100644 rust/kernel/io/poll.rs create mode 100644 rust/kernel/io/register.rs create mode 100644 rust/kernel/io/resource.rs create mode 100644 rust/kernel/ioctl.rs create mode 100644 rust/kernel/iommu/mod.rs create mode 100644 rust/kernel/iommu/pgtable.rs create mode 100644 rust/kernel/iov.rs create mode 100644 rust/kernel/irq.rs create mode 100644 rust/kernel/irq/flags.rs create mode 100644 rust/kernel/irq/request.rs create mode 100644 rust/kernel/jump_label.rs create mode 100644 rust/kernel/kunit.rs create mode 100644 rust/kernel/lib.rs create mode 100644 rust/kernel/list.rs create mode 100644 rust/kernel/list/arc.rs create mode 100644 rust/kernel/list/arc_field.rs create mode 100644 rust/kernel/list/impl_list_item_mod.rs create mode 100644 rust/kernel/maple_tree.rs create mode 100644 rust/kernel/miscdevice.rs create mode 100644 rust/kernel/mm.rs create mode 100644 rust/kernel/mm/mmput_async.rs create mode 100644 rust/kernel/mm/virt.rs create mode 100644 rust/kernel/module.rs create mode 100644 rust/kernel/module_param.rs create mode 100644 rust/kernel/net/mod.rs create mode 100644 rust/kernel/net/netlink.rs create mode 100644 rust/kernel/net/phy.rs create mode 100644 rust/kernel/net/phy/reg.rs create mode 100644 rust/kernel/num.rs create mode 100644 rust/kernel/num/bounded.rs create mode 100644 rust/kernel/num/casts.rs create mode 100644 rust/kernel/of.rs create mode 100644 rust/kernel/opp.rs create mode 100644 rust/kernel/page.rs create mode 100644 rust/kernel/pci.rs create mode 100644 rust/kernel/pci/id.rs create mode 100644 rust/kernel/pci/io.rs create mode 100644 rust/kernel/pci/irq.rs create mode 100644 rust/kernel/pid_namespace.rs create mode 100644 rust/kernel/platform.rs create mode 100644 rust/kernel/prelude.rs create mode 100644 rust/kernel/print.rs create mode 100644 rust/kernel/processor.rs create mode 100644 rust/kernel/ptr.rs create mode 100644 rust/kernel/ptr/projection.rs create mode 100644 rust/kernel/pwm.rs create mode 100644 rust/kernel/rbtree.rs create mode 100644 rust/kernel/regulator.rs create mode 100644 rust/kernel/revocable.rs create mode 100644 rust/kernel/safety.rs create mode 100644 rust/kernel/scatterlist.rs create mode 100644 rust/kernel/security.rs create mode 100644 rust/kernel/seq_file.rs create mode 100644 rust/kernel/serdev.rs create mode 100644 rust/kernel/sizes.rs create mode 100644 rust/kernel/soc.rs create mode 100644 rust/kernel/std_vendor.rs create mode 100644 rust/kernel/str.rs create mode 100644 rust/kernel/str/parse_int.rs create mode 100644 rust/kernel/sync.rs create mode 100644 rust/kernel/sync/arc.rs create mode 100644 rust/kernel/sync/arc/std_vendor.rs create mode 100644 rust/kernel/sync/aref.rs create mode 100644 rust/kernel/sync/atomic.rs create mode 100644 rust/kernel/sync/atomic/internal.rs create mode 100644 rust/kernel/sync/atomic/ordering.rs create mode 100644 rust/kernel/sync/atomic/predefine.rs create mode 100644 rust/kernel/sync/barrier.rs create mode 100644 rust/kernel/sync/completion.rs create mode 100644 rust/kernel/sync/condvar.rs create mode 100644 rust/kernel/sync/lock.rs create mode 100644 rust/kernel/sync/lock/global.rs create mode 100644 rust/kernel/sync/lock/mutex.rs create mode 100644 rust/kernel/sync/lock/spinlock.rs create mode 100644 rust/kernel/sync/locked_by.rs create mode 100644 rust/kernel/sync/poll.rs create mode 100644 rust/kernel/sync/rcu.rs create mode 100644 rust/kernel/sync/refcount.rs create mode 100644 rust/kernel/sync/set_once.rs create mode 100644 rust/kernel/sync/srcu.rs create mode 100644 rust/kernel/task.rs create mode 100644 rust/kernel/time.rs create mode 100644 rust/kernel/time/delay.rs create mode 100644 rust/kernel/time/hrtimer.rs create mode 100644 rust/kernel/time/hrtimer/arc.rs create mode 100644 rust/kernel/time/hrtimer/pin.rs create mode 100644 rust/kernel/time/hrtimer/pin_mut.rs create mode 100644 rust/kernel/time/hrtimer/tbox.rs create mode 100644 rust/kernel/tracepoint.rs create mode 100644 rust/kernel/transmute.rs create mode 100644 rust/kernel/types.rs create mode 100644 rust/kernel/types/for_lt.rs create mode 100644 rust/kernel/uaccess.rs create mode 100644 rust/kernel/usb.rs create mode 100644 rust/kernel/workqueue.rs create mode 100644 rust/kernel/xarray.rs create mode 100644 rust/macros/concat_idents.rs create mode 100644 rust/macros/export.rs create mode 100644 rust/macros/fmt.rs create mode 100644 rust/macros/for_lt.rs create mode 100644 rust/macros/helpers.rs create mode 100644 rust/macros/kunit.rs create mode 100644 rust/macros/lib.rs create mode 100644 rust/macros/module.rs create mode 100644 rust/macros/paste.rs create mode 100644 rust/macros/vtable.rs create mode 100644 rust/pin-init/CONTRIBUTING.md create mode 100644 rust/pin-init/README.md create mode 100644 rust/pin-init/examples/big_struct_in_place.rs create mode 100644 rust/pin-init/examples/error.rs create mode 100644 rust/pin-init/examples/linked_list.rs create mode 100644 rust/pin-init/examples/mutex.rs create mode 100644 rust/pin-init/examples/pthread_mutex.rs create mode 100644 rust/pin-init/examples/static_init.rs create mode 100644 rust/pin-init/internal/src/diagnostics.rs create mode 100644 rust/pin-init/internal/src/init.rs create mode 100644 rust/pin-init/internal/src/lib.rs create mode 100644 rust/pin-init/internal/src/pin_data.rs create mode 100644 rust/pin-init/internal/src/pinned_drop.rs create mode 100644 rust/pin-init/internal/src/zeroable.rs create mode 100644 rust/pin-init/src/__internal.rs create mode 100644 rust/pin-init/src/alloc.rs create mode 100644 rust/pin-init/src/lib.rs create mode 100644 rust/proc-macro2/README.md create mode 100644 rust/proc-macro2/detection.rs create mode 100644 rust/proc-macro2/extra.rs create mode 100644 rust/proc-macro2/fallback.rs create mode 100644 rust/proc-macro2/lib.rs create mode 100644 rust/proc-macro2/location.rs create mode 100644 rust/proc-macro2/marker.rs create mode 100644 rust/proc-macro2/parse.rs create mode 100644 rust/proc-macro2/probe.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_file.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_location.rs create mode 100644 rust/proc-macro2/rcvec.rs create mode 100644 rust/proc-macro2/wrapper.rs create mode 100644 rust/quote/README.md create mode 100644 rust/quote/ext.rs create mode 100644 rust/quote/format.rs create mode 100644 rust/quote/ident_fragment.rs create mode 100644 rust/quote/lib.rs create mode 100644 rust/quote/runtime.rs create mode 100644 rust/quote/spanned.rs create mode 100644 rust/quote/to_tokens.rs create mode 100644 rust/syn/README.md create mode 100644 rust/syn/attr.rs create mode 100644 rust/syn/bigint.rs create mode 100644 rust/syn/buffer.rs create mode 100644 rust/syn/classify.rs create mode 100644 rust/syn/custom_keyword.rs create mode 100644 rust/syn/custom_punctuation.rs create mode 100644 rust/syn/data.rs create mode 100644 rust/syn/derive.rs create mode 100644 rust/syn/discouraged.rs create mode 100644 rust/syn/drops.rs create mode 100644 rust/syn/error.rs create mode 100644 rust/syn/export.rs create mode 100644 rust/syn/expr.rs create mode 100644 rust/syn/ext.rs create mode 100644 rust/syn/file.rs create mode 100644 rust/syn/fixup.rs create mode 100644 rust/syn/gen/clone.rs create mode 100644 rust/syn/gen/debug.rs create mode 100644 rust/syn/gen/eq.rs create mode 100644 rust/syn/gen/fold.rs create mode 100644 rust/syn/gen/hash.rs create mode 100644 rust/syn/gen/visit.rs create mode 100644 rust/syn/gen/visit_mut.rs create mode 100644 rust/syn/generics.rs create mode 100644 rust/syn/group.rs create mode 100644 rust/syn/ident.rs create mode 100644 rust/syn/item.rs create mode 100644 rust/syn/lib.rs create mode 100644 rust/syn/lifetime.rs create mode 100644 rust/syn/lit.rs create mode 100644 rust/syn/lookahead.rs create mode 100644 rust/syn/mac.rs create mode 100644 rust/syn/macros.rs create mode 100644 rust/syn/meta.rs create mode 100644 rust/syn/op.rs create mode 100644 rust/syn/parse.rs create mode 100644 rust/syn/parse_macro_input.rs create mode 100644 rust/syn/parse_quote.rs create mode 100644 rust/syn/pat.rs create mode 100644 rust/syn/path.rs create mode 100644 rust/syn/precedence.rs create mode 100644 rust/syn/print.rs create mode 100644 rust/syn/punctuated.rs create mode 100644 rust/syn/restriction.rs create mode 100644 rust/syn/scan_expr.rs create mode 100644 rust/syn/sealed.rs create mode 100644 rust/syn/span.rs create mode 100644 rust/syn/spanned.rs create mode 100644 rust/syn/stmt.rs create mode 100644 rust/syn/thread.rs create mode 100644 rust/syn/token.rs create mode 100644 rust/syn/tt.rs create mode 100644 rust/syn/ty.rs create mode 100644 rust/syn/verbatim.rs create mode 100644 rust/syn/whitespace.rs create mode 100644 rust/uapi/lib.rs create mode 100644 rust/uapi/uapi_helper.h create mode 100644 rust/zerocopy-derive/README.md create mode 100644 rust/zerocopy-derive/derive/from_bytes.rs create mode 100644 rust/zerocopy-derive/derive/into_bytes.rs create mode 100644 rust/zerocopy-derive/derive/known_layout.rs create mode 100644 rust/zerocopy-derive/derive/mod.rs create mode 100644 rust/zerocopy-derive/derive/try_from_bytes.rs create mode 100644 rust/zerocopy-derive/derive/unaligned.rs create mode 100644 rust/zerocopy-derive/lib.rs create mode 100644 rust/zerocopy-derive/repr.rs create mode 100644 rust/zerocopy-derive/util.rs create mode 100644 rust/zerocopy/README.md create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/as_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_size.rs create mode 100644 rust/zerocopy/benches/formats/coco_static_size.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/new_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_zeroed.rs create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_bytes.rs create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_prefix.rs create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_suffix.rs create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute.rs create mode 100644 rust/zerocopy/benches/transmute.x86-64 create mode 100644 rust/zerocopy/benches/transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_bytes.rs create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_prefix.rs create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_suffix.rs create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute.rs create mode 100644 rust/zerocopy/benches/try_transmute.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_size.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_static_size.rs create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64.mca create mode 100644 rust/zerocopy/rustdoc/style.css create mode 100644 rust/zerocopy/src/byte_slice.rs create mode 100644 rust/zerocopy/src/byteorder.rs create mode 100644 rust/zerocopy/src/deprecated.rs create mode 100644 rust/zerocopy/src/error.rs create mode 100644 rust/zerocopy/src/impls.rs create mode 100644 rust/zerocopy/src/layout.rs create mode 100644 rust/zerocopy/src/lib.rs create mode 100644 rust/zerocopy/src/macros.rs create mode 100644 rust/zerocopy/src/pointer/inner.rs create mode 100644 rust/zerocopy/src/pointer/invariant.rs create mode 100644 rust/zerocopy/src/pointer/mod.rs create mode 100644 rust/zerocopy/src/pointer/ptr.rs create mode 100644 rust/zerocopy/src/pointer/transmute.rs create mode 100644 rust/zerocopy/src/ref.rs create mode 100644 rust/zerocopy/src/split_at.rs create mode 100644 rust/zerocopy/src/util/macro_util.rs create mode 100644 rust/zerocopy/src/util/macros.rs create mode 100644 rust/zerocopy/src/util/mod.rs create mode 100644 rust/zerocopy/src/wrappers.rs (limited to 'rust') diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 000000000..d3829ffab --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 + +bindings_generated.rs +bindings_helpers_generated.rs +doctests_kernel_generated.rs +doctests_kernel_generated_kunit.c +uapi_generated.rs +exports_*_generated.h +doc/ +test/ diff --git a/rust/.kunitconfig b/rust/.kunitconfig new file mode 100644 index 000000000..9e72a5ab0 --- /dev/null +++ b/rust/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_RUST=y +CONFIG_RUST_KERNEL_DOCTESTS=y diff --git a/rust/Makefile b/rust/Makefile new file mode 100644 index 000000000..da1a7409d --- /dev/null +++ b/rust/Makefile @@ -0,0 +1,815 @@ +# SPDX-License-Identifier: GPL-2.0 + +# Where to place rustdoc generated documentation +rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc + +# Clean generated host directory +clean-files := host/ + +obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o +always-$(CONFIG_RUST) += exports_core_generated.h + +obj-$(CONFIG_RUST) += zerocopy.o + +ifdef CONFIG_RUST_INLINE_HELPERS +always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc +else +obj-$(CONFIG_RUST) += helpers/helpers.o +always-$(CONFIG_RUST) += exports_helpers_generated.h +endif +# Missing prototypes are expected in the helpers since these are exported +# for Rust only, thus there is no header nor prototypes. +CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations + +always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs +obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o +always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h + +always-$(CONFIG_RUST) += uapi/uapi_generated.rs +obj-$(CONFIG_RUST) += uapi.o + +ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW +obj-$(CONFIG_RUST) += build_error.o +else +always-$(CONFIG_RUST) += build_error.o +endif + +obj-$(CONFIG_RUST) += exports.o + +always-$(CONFIG_RUST) += host/libproc_macro2.rlib host/libquote.rlib host/libsyn.rlib + +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c + +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o + +always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs +ifndef CONFIG_UML +always-$(subst y,$(CONFIG_RUST),$(CONFIG_BUG)) += kernel/generated_arch_warn_asm.rs kernel/generated_arch_reachable_asm.rs +endif + +# Avoids running `$(RUSTC)` when it may not be available. +ifdef CONFIG_RUST + +procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - /dev/null +endif + +cfgs-to-flags = $(patsubst %,--cfg='%',$1) + +core-cfgs := \ + no_fp_fmt_parse + +core-edition := $(if $(call rustc-min-version,108700),2024,2021) + +core-skip_flags := \ + --edition=2021 \ + -Wunreachable_pub + +core-flags := \ + --edition=$(core-edition) \ + $(call cfgs-to-flags,$(core-cfgs)) + +zerocopy-cfgs := \ + no_fp_fmt_parse + +zerocopy-flags := \ + --cap-lints=allow \ + $(call cfgs-to-flags,$(zerocopy-cfgs)) + +zerocopy-envs := \ + CARGO_PKG_VERSION=0.8.54 + +proc_macro2-cfgs := \ + feature="proc-macro" \ + wrap_proc_macro \ + $(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location) + +proc_macro2-flags := \ + --cap-lints=allow \ + $(call cfgs-to-flags,$(proc_macro2-cfgs)) + +quote-cfgs := \ + feature="proc-macro" + +quote-skip_flags := \ + --edition=2021 + +quote-flags := \ + --edition=2018 \ + --cap-lints=allow \ + --extern proc_macro2 \ + $(call cfgs-to-flags,$(quote-cfgs)) + +# `extra-traits`, `fold` and `visit` may be enabled if needed. +syn-cfgs := \ + feature="clone-impls" \ + feature="derive" \ + feature="full" \ + feature="parsing" \ + feature="printing" \ + feature="proc-macro" \ + feature="visit" \ + feature="visit-mut" + +syn-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + $(call cfgs-to-flags,$(syn-cfgs)) + +zerocopy_derive-cfgs := \ + zerocopy_unstable_linux + +zerocopy_derive-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + --extern syn \ + $(call cfgs-to-flags,$(zerocopy_derive-cfgs)) + +pin_init_internal-cfgs := \ + kernel USE_RUSTC_FEATURES + +pin_init_internal-flags := \ + --extern proc_macro2 \ + --extern quote \ + --extern syn \ + $(call cfgs-to-flags,$(pin_init_internal-cfgs)) + +pin_init-cfgs := \ + kernel USE_RUSTC_FEATURES + +pin_init-flags := \ + --extern pin_init_internal \ + --extern macros \ + $(call cfgs-to-flags,$(pin_init-cfgs)) + +# `rustdoc` did not save the target modifiers, thus workaround for +# the time being (https://github.com/rust-lang/rust/issues/144521). +rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18) + +# Similarly, for doctests (https://github.com/rust-lang/rust/issues/146465). +doctests_modifiers_workaround := $(rustdoc_modifiers_workaround)$(if $(call rustc-min-version,109100),$(comma)sanitizer) + +quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< + cmd_rustdoc = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ + $(rustc_target_flags) -L$(objtree)/$(obj)$(if $(rustdoc_host),/host) \ + -Zunstable-options --generate-link-to-definition \ + --output $(rustdoc_output) \ + --crate-name $(subst rustdoc-,,$@) \ + $(rustdoc_modifiers_workaround) \ + $(if $(rustdoc_host),,--sysroot=/dev/null) \ + @$(objtree)/include/generated/rustc_cfg $< + +# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute +# can be used to specify a custom logo. However: +# - The given value is used as-is, thus it cannot be relative or a local file +# (unlike the non-custom case) since the generated docs have subfolders. +# - It requires adding it to every crate. +# - It requires changing `core` which comes from the sysroot. +# +# Using `-Zcrate-attr` would solve the last two points, but not the first. +# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new +# command-like flags to solve the issue. Meanwhile, we use the non-custom case +# and then retouch the generated files. +rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ + rustdoc-kernel rustdoc-pin_init rustdoc-zerocopy rustdoc-zerocopy_derive + $(Q)grep -Ehro '::g' \ + -e 's:::g' + $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \ + echo ".logo-container > img { object-fit: contain; }" >> $$f; done + +rustdoc-proc_macro2: private rustdoc_host = yes +rustdoc-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rustdoc-proc_macro2: $(src)/proc-macro2/lib.rs rustdoc-clean FORCE + +$(call if_changed,rustdoc) + +rustdoc-quote: private rustdoc_host = yes +rustdoc-quote: private rustc_target_flags = $(quote-flags) +rustdoc-quote: private skip_flags = $(quote-skip_flags) +rustdoc-quote: $(src)/quote/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE + +$(call if_changed,rustdoc) + +rustdoc-syn: private rustdoc_host = yes +rustdoc-syn: private rustc_target_flags = $(syn-flags) +rustdoc-syn: $(src)/syn/lib.rs rustdoc-clean rustdoc-quote FORCE + +$(call if_changed,rustdoc) + +rustdoc-zerocopy_derive: private rustdoc_host = yes +rustdoc-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rustdoc-clean rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-macros: private rustdoc_host = yes +rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ + --extern proc_macro --extern proc_macro2 --extern quote --extern syn +rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 \ + rustdoc-quote rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-core: private skip_flags = $(core-skip_flags) +rustdoc-core: private rustc_target_flags = $(core-flags) +rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs rustdoc-clean FORCE + +$(call if_changed,rustdoc) + +# Even if `rustdoc` targets are not kernel objects, they should still be +# treated as such so that we pass the same flags. Otherwise, for instance, +# `rustdoc` will complain about missing sanitizer flags causing an ABI mismatch. +rustdoc-compiler_builtins: private is-kernel-object := y +rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rustdoc-zerocopy: private is-kernel-object := y +rustdoc-zerocopy: private rustc_target_flags = $(zerocopy-flags) \ + --extend-css $(src)/zerocopy/rustdoc/style.css +rustdoc-zerocopy: $(src)/zerocopy/src/lib.rs rustdoc-clean rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-ffi: private is-kernel-object := y +rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-pin_init_internal: private rustdoc_host = yes +rustdoc-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ + rustdoc-clean rustdoc-proc_macro2 rustdoc-quote rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-pin_init: private rustdoc_host = yes +rustdoc-pin_init: private rustc_target_flags = $(pin_init-flags) \ + --extern pin_init_internal=$(objtree)/$(obj)/$(libpin_init_internal_name) \ + --extern alloc --cfg feature=\"alloc\" +rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ + rustdoc-macros FORCE + +$(call if_changed,rustdoc) + +rustdoc-kernel: private is-kernel-object := y +rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ + --extern build_error --extern macros \ + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive +rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ + rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \ + $(obj)/bindings.o FORCE + +$(call if_changed,rustdoc) + +rustdoc-clean: FORCE + $(Q)rm -rf $(rustdoc_output) + +quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< + cmd_rustc_test_library = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + @$(objtree)/include/generated/rustc_cfg \ + --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ + --out-dir $(objtree)/$(obj)/test --cfg testlib \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< + +rusttestlib-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rusttestlib-zerocopy: private rustc_target_flags = $(zerocopy-flags) +rusttestlib-zerocopy: $(src)/zerocopy/src/lib.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-build_error: $(src)/build_error.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-ffi: $(src)/ffi.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rusttestlib-proc_macro2: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-quote: private skip_flags = $(quote-skip_flags) +rusttestlib-quote: private rustc_target_flags = $(quote-flags) +rusttestlib-quote: $(src)/quote/lib.rs rusttestlib-proc_macro2 FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-syn: private rustc_target_flags = $(syn-flags) +rusttestlib-syn: $(src)/syn/lib.rs rusttestlib-quote FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro +rusttestlib-zerocopy_derive: private rustc_test_library_proc = yes +rusttestlib-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-macros: private rustc_target_flags = --extern proc_macro \ + --extern proc_macro2 --extern quote --extern syn +rusttestlib-macros: private rustc_test_library_proc = yes +rusttestlib-macros: $(src)/macros/lib.rs \ + rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \ + --extern proc_macro +rusttestlib-pin_init_internal: private rustc_test_library_proc = yes +rusttestlib-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ + rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-pin_init: private rustc_target_flags = $(pin_init-flags) +rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ + rusttestlib-pin_init_internal $(obj)/$(libpin_init_internal_name) FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-kernel: private rustc_target_flags = --extern ffi \ + --extern build_error --extern macros --extern pin_init \ + --extern bindings --extern uapi \ + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib \ + --extern zerocopy_derive=$(objtree)/$(obj)/test/$(libzerocopy_derive_name) +rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ + rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ + $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-uapi: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test_library) + +quiet_cmd_rustdoc_test = RUSTDOC T $< + cmd_rustdoc_test = \ + $(rustc_target_envs) \ + RUST_MODFILE=test.rs \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(rust_common_flags) \ + -Zcrate-attr='feature(used_with_arg)' \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) $(rustdoc_test_target_flags) \ + $(rustdoc_test_quiet) \ + -L$(objtree)/$(obj)/test --output $(rustdoc_output) \ + --crate-name $(subst rusttest-,,$@) $< + +quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< + cmd_rustdoc_test_kernel = \ + rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ + mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(filter-out --remap-path-scope=%,$(rust_flags)) \ + -L$(objtree)/$(obj) --extern ffi --extern pin_init \ + --extern kernel --extern build_error --extern macros \ + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive \ + --no-run --crate-name kernel -Zunstable-options \ + --sysroot=/dev/null \ + $(doctests_modifiers_workaround) \ + --test-builder $(objtree)/scripts/rustdoc_test_builder \ + $< $(rustdoc_test_kernel_quiet); \ + $(objtree)/scripts/rustdoc_test_gen + +%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \ + $(src)/kernel/lib.rs $(obj)/kernel.o \ + $(objtree)/scripts/rustdoc_test_builder \ + $(objtree)/scripts/rustdoc_test_gen FORCE + +$(call if_changed,rustdoc_test_kernel) + +# We cannot use `-Zpanic-abort-tests` because some tests are dynamic, +# so for the moment we skip `-Cpanic=abort`. +quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< + cmd_rustc_test = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$@) $<; \ + $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \ + $(rustc_test_run_flags) + +rusttest: rusttest-macros + +rusttest-macros: private rustc_target_flags = --extern proc_macro \ + --extern macros --extern kernel --extern pin_init \ + --extern proc_macro2 --extern quote --extern syn +rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro +rusttest-macros: $(src)/macros/lib.rs \ + rusttestlib-macros rusttestlib-kernel rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test) + +$(call if_changed,rustdoc_test) + +ifdef CONFIG_CC_IS_CLANG +bindgen_c_flags = $(c_flags) +else +# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC +# plugin backend and/or the Clang driver would be perfectly compatible with GCC. +# +# For the moment, here we are tweaking the flags on the fly. This is a hack, +# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT` +# if we end up using one of those structs). +bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ + -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \ + -mindirect-branch=thunk-extern -mindirect-branch-register \ + -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \ + -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \ + -mno-pointers-to-nested-functions -mno-string \ + -mno-strict-align -mstrict-align -mdirect-extern-access \ + -mexplicit-relocs -mno-check-zero-division \ + -fconserve-stack -falign-jumps=% -falign-loops=% \ + -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \ + -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \ + -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \ + -fzero-call-used-regs=% -fno-stack-clash-protection \ + -fno-inline-functions-called-once -fsanitize=bounds-strict \ + -fstrict-flex-arrays=% -fmin-function-alignment=% \ + -fzero-init-padding-bits=% -mno-fdpic \ + -fdiagnostics-show-context -fdiagnostics-show-context=% \ + --param=% --param asan-% -fno-isolate-erroneous-paths-dereference \ + -ffixed-r2 -mmultiple -mno-readonly-in-sdata + +# Derived from `scripts/Makefile.clang`. +BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET_arm64 := aarch64-linux-gnu +BINDGEN_TARGET_arm := arm-linux-gnueabi +BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf +BINDGEN_TARGET_s390 := s390x-linux-gnu +# This is only for i386 UM builds, which need the 32-bit target not -m32 +BINDGEN_TARGET_i386 := i386-linux-gnu + +ifdef CONFIG_PPC64 +BINDGEN_TARGET_powerpc := powerpc64le-linux-gnu +else +BINDGEN_TARGET_powerpc := powerpc-linux-gnu +endif + +BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) +BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) + +# All warnings are inhibited since GCC builds are very experimental, +# many GCC warnings are not supported by Clang, they may only appear in +# some configurations, with new GCC versions, etc. +bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET) + +# Auto variable zero-initialization requires an additional special option with +# clang that is going to be removed sometime in the future (likely in +# clang-18), so make sure to pass this option only if clang supports it +# (libclang major version < 16). +# +# https://github.com/llvm/llvm-project/issues/44842 +# https://github.com/llvm/llvm-project/blob/llvmorg-16.0.0-rc2/clang/docs/ReleaseNotes.rst#deprecated-compiler-flags +ifdef CONFIG_INIT_STACK_ALL_ZERO +libclang_maj_ver=$(shell $(BINDGEN) $(srctree)/scripts/rust_is_available_bindgen_libclang.h 2>&1 | sed -ne 's/.*clang version \([0-9]*\).*/\1/p') +ifeq ($(shell expr $(libclang_maj_ver) \< 16), 1) +bindgen_extra_c_flags += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang +endif +endif + +bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \ + $(bindgen_extra_c_flags) +endif + +ifdef CONFIG_LTO +bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags)) +else +bindgen_c_flags_lto = $(bindgen_c_flags) +endif + +# `-fno-builtin` is passed to avoid `bindgen` from using `clang` builtin +# prototypes for functions like `memcpy` -- if this flag is not passed, +# `bindgen`-generated prototypes use `c_ulong` or `c_uint` depending on +# architecture instead of generating `usize`. +bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ + +# `--rust-target` points to our minimum supported Rust version. +quiet_cmd_bindgen = BINDGEN $@ + cmd_bindgen = \ + $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ + --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ + --no-debug '.*' --enable-function-attribute-detection \ + -o $@ -- $(bindgen_c_flags_final) -DMODULE \ + $(bindgen_target_cflags) $(bindgen_target_extra) + +$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) +$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@ +$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + +$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) +$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + +# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn +# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here +# given it is `libclang`; but for consistency, future Clang changes and/or +# a potential future GCC backend for `bindgen`, we disable it too. +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \ + --blocklist-type '.*' --allowlist-var '' \ + --allowlist-function 'rust_helper_.*' +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \ + -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@ +$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE + $(call if_changed_dep,bindgen) + +quiet_cmd_rust_helper = HELPER $@ + cmd_rust_helper = \ + $(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) \ + -c -g0 $< -emit-llvm -o $@ + +$(obj)/helpers/helpers.bc: private part-of-builtin := y +$(obj)/helpers/helpers_module.bc: private part-of-module := y +$(obj)/helpers/helpers.bc $(obj)/helpers/helpers_module.bc: $(src)/helpers/helpers.c FORCE + +$(call if_changed_dep,rust_helper) + +rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' + +quiet_cmd_exports = EXPORTS $@ + cmd_exports = \ + $(call rust_exports,$<,"EXPORT_SYMBOL_RUST_GPL(%s);\n") > $@ + +$(obj)/exports_core_generated.h: $(obj)/core.o FORCE + $(call if_changed,exports) + +# Even though Rust kernel modules should never use the bindings directly, +# symbols from the `bindings` crate and the C helpers need to be exported +# because Rust generics and inlined functions may not get their code generated +# in the crate where they are defined. Other helpers, called from non-inline +# functions, may not be exported, in principle. However, in general, the Rust +# compiler does not guarantee codegen will be performed for a non-inline +# function either. Therefore, we export all symbols from helpers and bindings. +# In the future, this may be revisited to reduce the number of exports after +# the compiler is informed about the places codegen is required. +$(obj)/exports_helpers_generated.h: $(obj)/helpers/helpers.o FORCE + $(call if_changed,exports) + +$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE + $(call if_changed,exports) + +$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE + $(call if_changed,exports) + +quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@ + cmd_rustc_procmacrolibrary = \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ + --out-dir $(objtree)/$(obj)/host -L$(objtree)/$(obj)/host \ + --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $< + +$(obj)/host/libproc_macro2.rlib: private skip_clippy = 1 +$(obj)/host/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) +$(obj)/host/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +$(obj)/host/libquote.rlib: private skip_clippy = 1 +$(obj)/host/libquote.rlib: private skip_flags = $(quote-skip_flags) +$(obj)/host/libquote.rlib: private rustc_target_flags = $(quote-flags) +$(obj)/host/libquote.rlib: $(src)/quote/lib.rs $(obj)/host/libproc_macro2.rlib FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +$(obj)/host/libsyn.rlib: private skip_clippy = 1 +$(obj)/host/libsyn.rlib: private rustc_target_flags = $(syn-flags) +$(obj)/host/libsyn.rlib: $(src)/syn/lib.rs $(obj)/host/libquote.rlib FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@ + cmd_rustc_procmacro = \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) $(rust_common_flags) $(rustc_target_flags) \ + -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ + -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ + --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ + --crate-type proc-macro -L$(objtree)/$(obj)/host \ + --crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \ + @$(objtree)/include/generated/rustc_cfg $< + +# Procedural macros can only be used with the `rustc` that compiled it. +$(obj)/$(libzerocopy_derive_name): private skip_clippy = 1 +$(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags) +$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +$(obj)/$(libmacros_name): private rustc_target_flags = \ + --extern proc_macro2 --extern quote --extern syn +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +$(obj)/$(libpin_init_internal_name): private rustc_target_flags = $(pin_init_internal-flags) +$(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \ + $(obj)/host/libproc_macro2.rlib $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +# `rustc` requires `-Zunstable-options` to use custom target specifications +# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). +quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ + cmd_rustc_library = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ + --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \ + --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ + --crate-type rlib -L$(objtree)/$(obj) \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ + --sysroot=/dev/null \ + -Zunstable-options \ + $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ + $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ + $(cmd_objtool) + +rust-analyzer: + $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ + --cfgs='core=$(core-cfgs)' $(core-edition) \ + --cfgs='zerocopy=$(zerocopy-cfgs)' \ + --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ + --cfgs='quote=$(quote-cfgs)' \ + --cfgs='syn=$(syn-cfgs)' \ + --cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \ + --cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \ + --cfgs='pin_init=$(pin_init-cfgs)' \ + --envs='zerocopy=$(zerocopy-envs)' \ + $(realpath $(srctree)) $(realpath $(objtree)) \ + $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ + > rust-project.json + +redirect-intrinsics = \ + __addsf3 __eqsf2 __extendsfdf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __truncdfsf2 __unordsf2 \ + __adddf3 __eqdf2 __ledf2 __ltdf2 __muldf3 __unorddf2 \ + __muloti4 __multi3 \ + __udivmodti4 __udivti3 __umodti3 + +ifdef CONFIG_ARM + # Add eabi initrinsics for ARM 32-bit + redirect-intrinsics += \ + __aeabi_fadd __aeabi_fmul __aeabi_fcmpeq __aeabi_fcmple __aeabi_fcmplt __aeabi_fcmpun \ + __aeabi_dadd __aeabi_dmul __aeabi_dcmple __aeabi_dcmplt __aeabi_dcmpun \ + __aeabi_uldivmod +endif +ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) + # These intrinsics are defined for ARM64 and RISCV64 + redirect-intrinsics += \ + __ashrti3 \ + __ashlti3 __lshrti3 +endif +ifdef CONFIG_PPC32 + redirect-intrinsics += \ + __udivdi3 __umoddi3 +endif + +ifdef CONFIG_MODVERSIONS +cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \ + $(call rust_exports,$@,"%s\n") | \ + scripts/gendwarfksyms/gendwarfksyms \ + $(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \ + $(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \ + $@ >> $(dot-target).cmd) +endif + +define rule_rustc_library + $(call cmd_and_fixdep,rustc_library) + $(call cmd,gen_objtooldep) + $(call cmd,gendwarfksyms) +endef + +define rule_rust_cc_library + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + $(call cmd,gendwarfksyms) +endef + +# helpers.o uses the same export mechanism as Rust libraries, so ensure symbol +# versions are calculated for the helpers too. +$(obj)/helpers/helpers.o: $(src)/helpers/helpers.c $(recordmcount_source) FORCE + +$(call if_changed_rule,rust_cc_library) + +# Disable symbol versioning for exports.o to avoid conflicts with the actual +# symbol versions generated from Rust objects. +$(obj)/exports.o: private skip_gendwarfksyms = 1 + +$(obj)/core.o: private skip_clippy = 1 +$(obj)/core.o: private skip_flags = $(core-skip_flags) +$(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) +$(obj)/core.o: private rustc_target_flags = $(core-flags) +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \ + $(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE + +$(call if_changed_rule,rustc_library) +ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),) +$(obj)/core.o: scripts/target.json +endif +KCOV_INSTRUMENT_core.o := n + +$(obj)/compiler_builtins.o: private skip_gendwarfksyms = 1 +$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' +$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/zerocopy.o: private skip_clippy = 1 +$(obj)/zerocopy.o: private skip_gendwarfksyms = 1 +$(obj)/zerocopy.o: private rustc_target_envs := $(zerocopy-envs) +$(obj)/zerocopy.o: private rustc_target_flags = $(zerocopy-flags) +$(obj)/zerocopy.o: $(src)/zerocopy/src/lib.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/pin_init.o: private skip_gendwarfksyms = 1 +$(obj)/pin_init.o: private rustc_target_flags = $(pin_init-flags) +$(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \ + $(obj)/$(libpin_init_internal_name) $(obj)/$(libmacros_name) FORCE + +$(call if_changed_rule,rustc_library) + +# Even if normally `build_error` is not a kernel object, it should still be +# treated as such so that we pass the same flags. Otherwise, for instance, +# `rustc` will complain about missing sanitizer flags causing an ABI mismatch. +$(obj)/build_error.o: private is-kernel-object := y +$(obj)/build_error.o: private skip_gendwarfksyms = 1 +$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/ffi.o: private skip_gendwarfksyms = 1 +$(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/bindings.o: private rustc_target_flags = --extern ffi --extern pin_init +$(obj)/bindings.o: $(src)/bindings/lib.rs \ + $(obj)/ffi.o \ + $(obj)/pin_init.o \ + $(obj)/bindings/bindings_generated.rs \ + $(obj)/bindings/bindings_helpers_generated.rs FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/uapi.o: private rustc_target_flags = --extern ffi --extern pin_init +$(obj)/uapi.o: private skip_gendwarfksyms = 1 +$(obj)/uapi.o: $(src)/uapi/lib.rs \ + $(obj)/ffi.o \ + $(obj)/pin_init.o \ + $(obj)/uapi/uapi_generated.rs FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \ + --extern build_error --extern macros --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ + $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o \ + $(obj)/zerocopy.o $(obj)/$(libzerocopy_derive_name) FORCE + +$(call if_changed_rule,rustc_library) + +ifdef CONFIG_JUMP_LABEL +$(obj)/kernel.o: $(obj)/kernel/generated_arch_static_branch_asm.rs +endif +ifndef CONFIG_UML +ifdef CONFIG_BUG +$(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generated_arch_reachable_asm.rs +endif +endif + +ifdef CONFIG_RUST_INLINE_HELPERS +$(obj)/kernel.o: private link_helper = 1 +$(obj)/kernel.o: $(obj)/helpers/helpers.bc +endif + +endif # CONFIG_RUST diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters new file mode 100644 index 000000000..8402b0c93 --- /dev/null +++ b/rust/bindgen_parameters @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: GPL-2.0 + +# We want to map these types to `isize`/`usize` manually, instead of +# define them as `int`/`long` depending on platform bitwidth. +--blocklist-type __kernel_s?size_t +--blocklist-type __kernel_ptrdiff_t + +--opaque-type xregs_state +--opaque-type desc_struct +--opaque-type arch_lbr_state +--opaque-type local_apic + +# Packed type cannot transitively contain a `#[repr(align)]` type. +--opaque-type alt_instr +--opaque-type x86_msi_data +--opaque-type x86_msi_addr_lo +# s390-only: same packed/align issue as above (E0588). +--opaque-type lowcore +--opaque-type tod_clock +--opaque-type tpi_info +--opaque-type uv_cb.* +--opaque-type uv_secret.* +--opaque-type zpci_fib + +# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust +# warning. We don't need to peek into it anyway. +--opaque-type spinlock + +# enums that appear in indirect function calls should specify a cfi type +--newtype-enum lru_status +--with-attribute-custom-enum=lru_status='#[cfi_encoding="10lru_status"]' + +# `seccomp`'s comment gets understood as a doctest +--no-doc-comments + +# These functions use the `__preserve_most` calling convention, which neither bindgen +# nor Rust currently understand, and which Clang currently declares to be unstable. +--blocklist-function __list_.*_report + +# These constants are sometimes not recognized by bindgen depending on config. +# We use const helpers to aid bindgen, to avoid conflicts when constants are +# recognized, block generation of the non-helper constants. +--blocklist-item ARCH_SLAB_MINALIGN +--blocklist-item ARCH_KMALLOC_MINALIGN +--blocklist-item VM_MERGEABLE +--blocklist-item VM_READ +--blocklist-item VM_WRITE +--blocklist-item VM_EXEC +--blocklist-item VM_SHARED +--blocklist-item VM_MAYREAD +--blocklist-item VM_MAYWRITE +--blocklist-item VM_MAYEXEC +--blocklist-item VM_MAYEXEC +--blocklist-item VM_PFNMAP +--blocklist-item VM_IO +--blocklist-item VM_DONTCOPY +--blocklist-item VM_DONTEXPAND +--blocklist-item VM_LOCKONFAULT +--blocklist-item VM_ACCOUNT +--blocklist-item VM_NORESERVE +--blocklist-item VM_HUGETLB +--blocklist-item VM_SYNC +--blocklist-item VM_ARCH_1 +--blocklist-item VM_WIPEONFORK +--blocklist-item VM_DONTDUMP +--blocklist-item VM_SOFTDIRTY +--blocklist-item VM_MIXEDMAP +--blocklist-item VM_HUGEPAGE +--blocklist-item VM_NOHUGEPAGE + +# Structs should implement `Zeroable` when all of their fields do. +--with-derive-custom-struct .*=MaybeZeroable +--with-derive-custom-union .*=MaybeZeroable diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h new file mode 100644 index 000000000..4b31aa7f4 --- /dev/null +++ b/rust/bindings/bindings_helper.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Header that contains the code (mostly headers) for which Rust bindings + * will be automatically generated by `bindgen`. + * + * Sorted alphabetically. + */ + +/* + * First, avoid forward references to `enum` types. + * + * This workarounds a `bindgen` issue with them: + * . + * + * Without this, the generated Rust type may be the wrong one (`i32`) or + * the proper one (typically `c_uint`) depending on how the headers are + * included, which in turn may depend on the particular kernel configuration + * or the architecture. + * + * The alternative would be to use casts and likely an + * `#[allow(clippy::unnecessary_cast)]` in the Rust source files. Instead, + * this approach allows us to keep the correct code in the source files and + * simply remove this section when the issue is fixed upstream and we bump + * the minimum `bindgen` version. + * + * This workaround may not be possible in some cases, depending on how the C + * headers are set up. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * The driver-core Rust code needs to know about some C driver-core private + * structures. + */ +#include <../../drivers/base/base.h> + +#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE) +// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`. +#include +#endif + +/* `bindgen` gets confused at certain things. */ +const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; +const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; +const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; +const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; +const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; +const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; +const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; +const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; +const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; + +const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; + +const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; +const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC1 = XA_FLAGS_ALLOC1; + +const vm_flags_t RUST_CONST_HELPER_VM_MERGEABLE = VM_MERGEABLE; +const vm_flags_t RUST_CONST_HELPER_VM_READ = VM_READ; +const vm_flags_t RUST_CONST_HELPER_VM_WRITE = VM_WRITE; +const vm_flags_t RUST_CONST_HELPER_VM_EXEC = VM_EXEC; +const vm_flags_t RUST_CONST_HELPER_VM_SHARED = VM_SHARED; +const vm_flags_t RUST_CONST_HELPER_VM_MAYREAD = VM_MAYREAD; +const vm_flags_t RUST_CONST_HELPER_VM_MAYWRITE = VM_MAYWRITE; +const vm_flags_t RUST_CONST_HELPER_VM_MAYEXEC = VM_MAYEXEC; +const vm_flags_t RUST_CONST_HELPER_VM_MAYSHARE = VM_MAYEXEC; +const vm_flags_t RUST_CONST_HELPER_VM_PFNMAP = VM_PFNMAP; +const vm_flags_t RUST_CONST_HELPER_VM_IO = VM_IO; +const vm_flags_t RUST_CONST_HELPER_VM_DONTCOPY = VM_DONTCOPY; +const vm_flags_t RUST_CONST_HELPER_VM_DONTEXPAND = VM_DONTEXPAND; +const vm_flags_t RUST_CONST_HELPER_VM_LOCKONFAULT = VM_LOCKONFAULT; +const vm_flags_t RUST_CONST_HELPER_VM_ACCOUNT = VM_ACCOUNT; +const vm_flags_t RUST_CONST_HELPER_VM_NORESERVE = VM_NORESERVE; +const vm_flags_t RUST_CONST_HELPER_VM_HUGETLB = VM_HUGETLB; +const vm_flags_t RUST_CONST_HELPER_VM_SYNC = VM_SYNC; +const vm_flags_t RUST_CONST_HELPER_VM_ARCH_1 = VM_ARCH_1; +const vm_flags_t RUST_CONST_HELPER_VM_WIPEONFORK = VM_WIPEONFORK; +const vm_flags_t RUST_CONST_HELPER_VM_DONTDUMP = VM_DONTDUMP; +const vm_flags_t RUST_CONST_HELPER_VM_SOFTDIRTY = VM_SOFTDIRTY; +const vm_flags_t RUST_CONST_HELPER_VM_MIXEDMAP = VM_MIXEDMAP; +const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE; +const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE; + +#if IS_ENABLED(CONFIG_GPU_BUDDY) +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TOPDOWN_ALLOCATION = GPU_BUDDY_TOPDOWN_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CONTIGUOUS_ALLOCATION = + GPU_BUDDY_CONTIGUOUS_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEAR_ALLOCATION = GPU_BUDDY_CLEAR_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; +#endif + +#if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) +#include "../../drivers/android/binder/rust_binder.h" +#include "../../drivers/android/binder/rust_binder_events.h" +#endif diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs new file mode 100644 index 000000000..439ab88a5 --- /dev/null +++ b/rust/bindings/lib.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bindings. +//! +//! Imports the generated bindings by `bindgen`. +//! +//! This crate may not be directly used. If you need a kernel C API that is +//! not ported or wrapped in the `kernel` crate, then do so first instead of +//! using this crate. + +#![no_std] +#![allow( + clippy::all, + missing_docs, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + improper_ctypes, + unreachable_pub, + unsafe_op_in_unsafe_fn +)] +#![feature(cfi_encoding)] + +#[allow(dead_code)] +#[allow(clippy::as_underscore)] +#[allow(clippy::cast_lossless)] +#[allow(clippy::ptr_as_ptr)] +#[allow(clippy::ref_as_ptr)] +#[allow(clippy::undocumented_unsafe_blocks)] +#[cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#[allow(unnecessary_transmutes)] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] +mod bindings_raw { + use pin_init::{MaybeZeroable, Zeroable}; + + // Manual definition for blocklisted types. + type __kernel_size_t = usize; + type __kernel_ssize_t = isize; + type __kernel_ptrdiff_t = isize; + + // `bindgen` doesn't automatically do this, see + // + // + // SAFETY: `__BindgenBitfieldUnit` is a newtype around `Storage`. + unsafe impl Zeroable for __BindgenBitfieldUnit where Storage: Zeroable {} + + // Use glob import here to expose all helpers. + // Symbols defined within the module will take precedence to the glob import. + pub use super::bindings_helper::*; + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_generated.rs" + )); +} + +// When both a directly exposed symbol and a helper exists for the same function, +// the directly exposed symbol is preferred and the helper becomes dead code, so +// ignore the warning here. +#[allow(dead_code)] +mod bindings_helper { + // Import the generated bindings for types. + use super::bindings_raw::*; + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_helpers_generated.rs" + )); +} + +pub use bindings_raw::*; + +pub const compat_ptr_ioctl: Option< + unsafe extern "C" fn(*mut file, ffi::c_uint, ffi::c_ulong) -> ffi::c_long, +> = { + #[cfg(CONFIG_COMPAT)] + { + Some(bindings_raw::compat_ptr_ioctl) + } + #[cfg(not(CONFIG_COMPAT))] + { + None + } +}; diff --git a/rust/build_error.rs b/rust/build_error.rs new file mode 100644 index 000000000..fa24eeef9 --- /dev/null +++ b/rust/build_error.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Build-time error. +//! +//! This crate provides a [const function][const-functions] `build_error`, which will panic in +//! compile-time if executed in [const context][const-context], and will cause a build error +//! if not executed at compile time and the optimizer does not optimise away the call. +//! +//! It is used by `build_assert!` in the kernel crate, allowing checking of +//! conditions that could be checked statically, but could not be enforced in +//! Rust yet (e.g. perform some checks in [const functions][const-functions], but those +//! functions could still be called in the runtime). +//! +//! For details on constant evaluation in Rust, please see the [Reference][const-eval]. +//! +//! [const-eval]: https://doc.rust-lang.org/reference/const_eval.html +//! [const-functions]: https://doc.rust-lang.org/reference/const_eval.html#const-functions +//! [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context + +#![no_std] + +/// Panics if executed in [const context][const-context], or triggers a build error if not. +/// +/// [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context +#[inline(never)] +#[cold] +#[export_name = "rust_build_error"] +#[track_caller] +pub const fn build_error(msg: &'static str) -> ! { + panic!("{}", msg); +} diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs new file mode 100644 index 000000000..fc6b54636 --- /dev/null +++ b/rust/compiler_builtins.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Our own `compiler_builtins`. +//! +//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`]. +//! Since we do not need the vast majority of them, we avoid the dependency +//! by providing this file. +//! +//! At the moment, some builtins are required that should not be. For instance, +//! [`core`] has 128-bit integers functionality which we should not be compiling +//! in. We will work with upstream [`core`] to provide feature flags to disable +//! the parts we do not need. For the moment, we define them to [`panic!`] at +//! runtime for simplicity to catch mistakes, instead of performing surgery +//! on `core.o`. +//! +//! In any case, all these symbols are weakened to ensure we do not override +//! those that may be provided by the rest of the kernel. +//! +//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins +//! [`compiler-rt`]: https://compiler-rt.llvm.org/ + +#![allow(internal_features)] +#![feature(compiler_builtins)] +#![compiler_builtins] +#![no_builtins] +#![no_std] + +macro_rules! define_panicking_intrinsics( + ($reason: tt, { $($ident: ident, )* }) => { + $( + #[doc(hidden)] + #[export_name = concat!("__rust", stringify!($ident))] + pub extern "C" fn $ident() { + panic!($reason); + } + )* + } +); + +define_panicking_intrinsics!("`f32` should not be used", { + __addsf3, + __eqsf2, + __extendsfdf2, + __gesf2, + __lesf2, + __ltsf2, + __mulsf3, + __nesf2, + __truncdfsf2, + __unordsf2, +}); + +define_panicking_intrinsics!("`f64` should not be used", { + __adddf3, + __eqdf2, + __ledf2, + __ltdf2, + __muldf3, + __unorddf2, +}); + +define_panicking_intrinsics!("`i128` should not be used", { + __ashrti3, + __muloti4, + __multi3, +}); + +define_panicking_intrinsics!("`u128` should not be used", { + __ashlti3, + __lshrti3, + __udivmodti4, + __udivti3, + __umodti3, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f32` should not be used", { + __aeabi_fadd, + __aeabi_fmul, + __aeabi_fcmpeq, + __aeabi_fcmple, + __aeabi_fcmplt, + __aeabi_fcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f64` should not be used", { + __aeabi_dadd, + __aeabi_dmul, + __aeabi_dcmple, + __aeabi_dcmplt, + __aeabi_dcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __aeabi_uldivmod, +}); + +#[cfg(target_arch = "powerpc")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __udivdi3, + __umoddi3, +}); + +// NOTE: if you are adding a new intrinsic here, you should also add it to +// `redirect-intrinsics` in `rust/Makefile`. diff --git a/rust/exports.c b/rust/exports.c new file mode 100644 index 000000000..1b52460b0 --- /dev/null +++ b/rust/exports.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A hack to export Rust symbols for loadable modules without having to redo + * the entire `include/linux/export.h` logic in Rust. + * + * This requires Rust's new/future `v0` mangling scheme because the default one + * ("legacy") uses invalid characters for C identifiers (thus we cannot use the + * `EXPORT_SYMBOL_*` macros). + * + * All symbols are exported as GPL-only to guarantee no GPL-only feature is + * accidentally exposed. + */ + +#include + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) + +#include "exports_core_generated.h" +#include "exports_bindings_generated.h" +#include "exports_kernel_generated.h" + +#ifndef CONFIG_RUST_INLINE_HELPERS +#include "exports_helpers_generated.h" +#endif + +// For modules using `rust/build_error.rs`. +#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW +EXPORT_SYMBOL_RUST_GPL(rust_build_error); +#endif diff --git a/rust/ffi.rs b/rust/ffi.rs new file mode 100644 index 000000000..f961e9728 --- /dev/null +++ b/rust/ffi.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Foreign function interface (FFI) types. +//! +//! This crate provides mapping from C primitive types to Rust ones. +//! +//! The Rust [`core`] crate provides [`core::ffi`], which maps integer types to the platform default +//! C ABI. The kernel does not use [`core::ffi`], so it can customise the mapping that deviates from +//! the platform default. + +#![no_std] + +macro_rules! alias { + ($($name:ident = $ty:ty;)*) => {$( + #[allow(non_camel_case_types, missing_docs)] + pub type $name = $ty; + + // Check size compatibility with `core`. + const _: () = assert!( + ::core::mem::size_of::<$name>() == ::core::mem::size_of::<::core::ffi::$name>() + ); + )*} +} + +alias! { + // `core::ffi::c_char` is either `i8` or `u8` depending on architecture. In the kernel, we use + // `-funsigned-char` so it's always mapped to `u8`. + c_char = u8; + + c_schar = i8; + c_uchar = u8; + + c_short = i16; + c_ushort = u16; + + c_int = i32; + c_uint = u32; + + // In the kernel, `intptr_t` is defined to be `long` in all platforms, so we can map the type to + // `isize`. + c_long = isize; + c_ulong = usize; + + c_longlong = i64; + c_ulonglong = u64; +} + +pub use core::ffi::c_void; + +pub use core::ffi::CStr; diff --git a/rust/helpers/atomic.c b/rust/helpers/atomic.c new file mode 100644 index 000000000..4b24eceef --- /dev/null +++ b/rust/helpers/atomic.c @@ -0,0 +1,1035 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-rust-atomic-helpers.sh +// DO NOT MODIFY THIS FILE DIRECTLY + +/* + * This file provides helpers for the various atomic functions for Rust. + */ +#ifndef _RUST_ATOMIC_API_H +#define _RUST_ATOMIC_API_H + +#include + +__rust_helper int +rust_helper_atomic_read(const atomic_t *v) +{ + return atomic_read(v); +} + +__rust_helper int +rust_helper_atomic_read_acquire(const atomic_t *v) +{ + return atomic_read_acquire(v); +} + +__rust_helper void +rust_helper_atomic_set(atomic_t *v, int i) +{ + atomic_set(v, i); +} + +__rust_helper void +rust_helper_atomic_set_release(atomic_t *v, int i) +{ + atomic_set_release(v, i); +} + +__rust_helper void +rust_helper_atomic_add(int i, atomic_t *v) +{ + atomic_add(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return(int i, atomic_t *v) +{ + return atomic_add_return(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_acquire(int i, atomic_t *v) +{ + return atomic_add_return_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_release(int i, atomic_t *v) +{ + return atomic_add_return_release(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_relaxed(int i, atomic_t *v) +{ + return atomic_add_return_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add(int i, atomic_t *v) +{ + return atomic_fetch_add(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_acquire(int i, atomic_t *v) +{ + return atomic_fetch_add_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_release(int i, atomic_t *v) +{ + return atomic_fetch_add_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_add_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_sub(int i, atomic_t *v) +{ + atomic_sub(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return(int i, atomic_t *v) +{ + return atomic_sub_return(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_acquire(int i, atomic_t *v) +{ + return atomic_sub_return_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_release(int i, atomic_t *v) +{ + return atomic_sub_return_release(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_relaxed(int i, atomic_t *v) +{ + return atomic_sub_return_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub(int i, atomic_t *v) +{ + return atomic_fetch_sub(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_acquire(int i, atomic_t *v) +{ + return atomic_fetch_sub_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_release(int i, atomic_t *v) +{ + return atomic_fetch_sub_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_sub_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_inc(atomic_t *v) +{ + atomic_inc(v); +} + +__rust_helper int +rust_helper_atomic_inc_return(atomic_t *v) +{ + return atomic_inc_return(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_acquire(atomic_t *v) +{ + return atomic_inc_return_acquire(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_release(atomic_t *v) +{ + return atomic_inc_return_release(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_relaxed(atomic_t *v) +{ + return atomic_inc_return_relaxed(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc(atomic_t *v) +{ + return atomic_fetch_inc(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_acquire(atomic_t *v) +{ + return atomic_fetch_inc_acquire(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_release(atomic_t *v) +{ + return atomic_fetch_inc_release(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_relaxed(atomic_t *v) +{ + return atomic_fetch_inc_relaxed(v); +} + +__rust_helper void +rust_helper_atomic_dec(atomic_t *v) +{ + atomic_dec(v); +} + +__rust_helper int +rust_helper_atomic_dec_return(atomic_t *v) +{ + return atomic_dec_return(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_acquire(atomic_t *v) +{ + return atomic_dec_return_acquire(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_release(atomic_t *v) +{ + return atomic_dec_return_release(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_relaxed(atomic_t *v) +{ + return atomic_dec_return_relaxed(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec(atomic_t *v) +{ + return atomic_fetch_dec(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_acquire(atomic_t *v) +{ + return atomic_fetch_dec_acquire(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_release(atomic_t *v) +{ + return atomic_fetch_dec_release(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_relaxed(atomic_t *v) +{ + return atomic_fetch_dec_relaxed(v); +} + +__rust_helper void +rust_helper_atomic_and(int i, atomic_t *v) +{ + atomic_and(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and(int i, atomic_t *v) +{ + return atomic_fetch_and(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_acquire(int i, atomic_t *v) +{ + return atomic_fetch_and_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_release(int i, atomic_t *v) +{ + return atomic_fetch_and_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_and_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_andnot(int i, atomic_t *v) +{ + atomic_andnot(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot(int i, atomic_t *v) +{ + return atomic_fetch_andnot(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + return atomic_fetch_andnot_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_release(int i, atomic_t *v) +{ + return atomic_fetch_andnot_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_andnot_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_or(int i, atomic_t *v) +{ + atomic_or(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or(int i, atomic_t *v) +{ + return atomic_fetch_or(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_acquire(int i, atomic_t *v) +{ + return atomic_fetch_or_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_release(int i, atomic_t *v) +{ + return atomic_fetch_or_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_or_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_xor(int i, atomic_t *v) +{ + atomic_xor(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor(int i, atomic_t *v) +{ + return atomic_fetch_xor(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_acquire(int i, atomic_t *v) +{ + return atomic_fetch_xor_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_release(int i, atomic_t *v) +{ + return atomic_fetch_xor_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_xor_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_xchg(atomic_t *v, int new) +{ + return atomic_xchg(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_acquire(atomic_t *v, int new) +{ + return atomic_xchg_acquire(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_release(atomic_t *v, int new) +{ + return atomic_xchg_release(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_relaxed(atomic_t *v, int new) +{ + return atomic_xchg_relaxed(v, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_acquire(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_release(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_acquire(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_release(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_sub_and_test(int i, atomic_t *v) +{ + return atomic_sub_and_test(i, v); +} + +__rust_helper bool +rust_helper_atomic_dec_and_test(atomic_t *v) +{ + return atomic_dec_and_test(v); +} + +__rust_helper bool +rust_helper_atomic_inc_and_test(atomic_t *v) +{ + return atomic_inc_and_test(v); +} + +__rust_helper bool +rust_helper_atomic_add_negative(int i, atomic_t *v) +{ + return atomic_add_negative(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_acquire(int i, atomic_t *v) +{ + return atomic_add_negative_acquire(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_release(int i, atomic_t *v) +{ + return atomic_add_negative_release(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_relaxed(int i, atomic_t *v) +{ + return atomic_add_negative_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + return atomic_fetch_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic_add_unless(atomic_t *v, int a, int u) +{ + return atomic_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic_inc_not_zero(atomic_t *v) +{ + return atomic_inc_not_zero(v); +} + +__rust_helper bool +rust_helper_atomic_inc_unless_negative(atomic_t *v) +{ + return atomic_inc_unless_negative(v); +} + +__rust_helper bool +rust_helper_atomic_dec_unless_positive(atomic_t *v) +{ + return atomic_dec_unless_positive(v); +} + +__rust_helper int +rust_helper_atomic_dec_if_positive(atomic_t *v) +{ + return atomic_dec_if_positive(v); +} + +__rust_helper s64 +rust_helper_atomic64_read(const atomic64_t *v) +{ + return atomic64_read(v); +} + +__rust_helper s64 +rust_helper_atomic64_read_acquire(const atomic64_t *v) +{ + return atomic64_read_acquire(v); +} + +__rust_helper void +rust_helper_atomic64_set(atomic64_t *v, s64 i) +{ + atomic64_set(v, i); +} + +__rust_helper void +rust_helper_atomic64_set_release(atomic64_t *v, s64 i) +{ + atomic64_set_release(v, i); +} + +__rust_helper void +rust_helper_atomic64_add(s64 i, atomic64_t *v) +{ + atomic64_add(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return(s64 i, atomic64_t *v) +{ + return atomic64_add_return(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_acquire(s64 i, atomic64_t *v) +{ + return atomic64_add_return_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_release(s64 i, atomic64_t *v) +{ + return atomic64_add_return_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_add_return_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_sub(s64 i, atomic64_t *v) +{ + atomic64_sub(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return(s64 i, atomic64_t *v) +{ + return atomic64_sub_return(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_acquire(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_release(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_inc(atomic64_t *v) +{ + atomic64_inc(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return(atomic64_t *v) +{ + return atomic64_inc_return(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_acquire(atomic64_t *v) +{ + return atomic64_inc_return_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_release(atomic64_t *v) +{ + return atomic64_inc_return_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_relaxed(atomic64_t *v) +{ + return atomic64_inc_return_relaxed(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc(atomic64_t *v) +{ + return atomic64_fetch_inc(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + return atomic64_fetch_inc_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_release(atomic64_t *v) +{ + return atomic64_fetch_inc_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_relaxed(atomic64_t *v) +{ + return atomic64_fetch_inc_relaxed(v); +} + +__rust_helper void +rust_helper_atomic64_dec(atomic64_t *v) +{ + atomic64_dec(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return(atomic64_t *v) +{ + return atomic64_dec_return(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_acquire(atomic64_t *v) +{ + return atomic64_dec_return_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_release(atomic64_t *v) +{ + return atomic64_dec_return_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_relaxed(atomic64_t *v) +{ + return atomic64_dec_return_relaxed(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec(atomic64_t *v) +{ + return atomic64_fetch_dec(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_acquire(atomic64_t *v) +{ + return atomic64_fetch_dec_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_release(atomic64_t *v) +{ + return atomic64_fetch_dec_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_relaxed(atomic64_t *v) +{ + return atomic64_fetch_dec_relaxed(v); +} + +__rust_helper void +rust_helper_atomic64_and(s64 i, atomic64_t *v) +{ + atomic64_and(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_andnot(s64 i, atomic64_t *v) +{ + atomic64_andnot(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_or(s64 i, atomic64_t *v) +{ + atomic64_or(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_xor(s64 i, atomic64_t *v) +{ + atomic64_xor(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_xchg(atomic64_t *v, s64 new) +{ + return atomic64_xchg(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_acquire(atomic64_t *v, s64 new) +{ + return atomic64_xchg_acquire(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_release(atomic64_t *v, s64 new) +{ + return atomic64_xchg_release(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_relaxed(atomic64_t *v, s64 new) +{ + return atomic64_xchg_relaxed(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_acquire(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_release(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_acquire(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_release(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_sub_and_test(s64 i, atomic64_t *v) +{ + return atomic64_sub_and_test(i, v); +} + +__rust_helper bool +rust_helper_atomic64_dec_and_test(atomic64_t *v) +{ + return atomic64_dec_and_test(v); +} + +__rust_helper bool +rust_helper_atomic64_inc_and_test(atomic64_t *v) +{ + return atomic64_inc_and_test(v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative(s64 i, atomic64_t *v) +{ + return atomic64_add_negative(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_acquire(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_acquire(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_release(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_release(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return atomic64_fetch_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return atomic64_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic64_inc_not_zero(atomic64_t *v) +{ + return atomic64_inc_not_zero(v); +} + +__rust_helper bool +rust_helper_atomic64_inc_unless_negative(atomic64_t *v) +{ + return atomic64_inc_unless_negative(v); +} + +__rust_helper bool +rust_helper_atomic64_dec_unless_positive(atomic64_t *v) +{ + return atomic64_dec_unless_positive(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_if_positive(atomic64_t *v) +{ + return atomic64_dec_if_positive(v); +} + +#endif /* _RUST_ATOMIC_API_H */ +// e4edb6174dd42a265284958f00a7cea7ddb464b1 diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c new file mode 100644 index 000000000..c267d5190 --- /dev/null +++ b/rust/helpers/atomic_ext.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#define GEN_READ_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read(type *ptr) \ +{ \ + return READ_ONCE(*ptr); \ +} + +#define GEN_SET_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set(type *ptr, type val) \ +{ \ + WRITE_ONCE(*ptr, val); \ +} + +#define GEN_READ_ACQUIRE_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read_acquire(type *ptr) \ +{ \ + return smp_load_acquire(ptr); \ +} + +#define GEN_SET_RELEASE_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set_release(type *ptr, type val)\ +{ \ + smp_store_release(ptr, val); \ +} + +#define GEN_READ_SET_HELPERS(tname, type) \ + GEN_READ_HELPER(tname, type) \ + GEN_SET_HELPER(tname, type) \ + GEN_READ_ACQUIRE_HELPER(tname, type) \ + GEN_SET_RELEASE_HELPER(tname, type) \ + +GEN_READ_SET_HELPERS(i8, s8) +GEN_READ_SET_HELPERS(i16, s16) +GEN_READ_SET_HELPERS(ptr, const void *) + +/* + * xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding xchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +#define GEN_XCHG_HELPER(tname, type, suffix) \ +__rust_helper type \ +rust_helper_atomic_##tname##_xchg##suffix(type *ptr, type new) \ +{ \ + return xchg##suffix(ptr, new); \ +} + +#define GEN_XCHG_HELPERS(tname, type) \ + GEN_XCHG_HELPER(tname, type, ) \ + GEN_XCHG_HELPER(tname, type, _acquire) \ + GEN_XCHG_HELPER(tname, type, _release) \ + GEN_XCHG_HELPER(tname, type, _relaxed) \ + +GEN_XCHG_HELPERS(i8, s8) +GEN_XCHG_HELPERS(i16, s16) +GEN_XCHG_HELPERS(ptr, const void *) + +/* + * try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding try_cmpxchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +#define GEN_TRY_CMPXCHG_HELPER(tname, type, suffix) \ +__rust_helper bool \ +rust_helper_atomic_##tname##_try_cmpxchg##suffix(type *ptr, type *old, type new)\ +{ \ + return try_cmpxchg##suffix(ptr, old, new); \ +} + +#define GEN_TRY_CMPXCHG_HELPERS(tname, type) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, ) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _acquire) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _release) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _relaxed) \ + +GEN_TRY_CMPXCHG_HELPERS(i8, s8) +GEN_TRY_CMPXCHG_HELPERS(i16, s16) +GEN_TRY_CMPXCHG_HELPERS(ptr, const void *) diff --git a/rust/helpers/auxiliary.c b/rust/helpers/auxiliary.c new file mode 100644 index 000000000..dd1c13084 --- /dev/null +++ b/rust/helpers/auxiliary.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void +rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev) +{ + return auxiliary_device_uninit(adev); +} + +__rust_helper void +rust_helper_auxiliary_device_delete(struct auxiliary_device *adev) +{ + return auxiliary_device_delete(adev); +} diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c new file mode 100644 index 000000000..dbc7a3017 --- /dev/null +++ b/rust/helpers/barrier.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + +__rust_helper void rust_helper_smp_mb(void) +{ + smp_mb(); +} + +__rust_helper void rust_helper_smp_wmb(void) +{ + smp_wmb(); +} + +__rust_helper void rust_helper_smp_rmb(void) +{ + smp_rmb(); +} diff --git a/rust/helpers/binder.c b/rust/helpers/binder.c new file mode 100644 index 000000000..a2327f1b3 --- /dev/null +++ b/rust/helpers/binder.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2025 Google LLC. + */ + +#include +#include + +__rust_helper unsigned long rust_helper_list_lru_count(struct list_lru *lru) +{ + return list_lru_count(lru); +} + +__rust_helper unsigned long rust_helper_list_lru_walk(struct list_lru *lru, + list_lru_walk_cb isolate, + void *cb_arg, + unsigned long nr_to_walk) +{ + return list_lru_walk(lru, isolate, cb_arg, nr_to_walk); +} + +__rust_helper void rust_helper_init_task_work(struct callback_head *twork, + task_work_func_t func) +{ + init_task_work(twork, func); +} diff --git a/rust/helpers/bitmap.c b/rust/helpers/bitmap.c new file mode 100644 index 000000000..e4e9f4361 --- /dev/null +++ b/rust/helpers/bitmap.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_bitmap_copy_and_extend(unsigned long *to, const unsigned long *from, + unsigned int count, unsigned int size) +{ + bitmap_copy_and_extend(to, from, count, size); +} diff --git a/rust/helpers/bitops.c b/rust/helpers/bitops.c new file mode 100644 index 000000000..271b8a712 --- /dev/null +++ b/rust/helpers/bitops.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper +void rust_helper___set_bit(unsigned long nr, unsigned long *addr) +{ + __set_bit(nr, addr); +} + +__rust_helper +void rust_helper___clear_bit(unsigned long nr, unsigned long *addr) +{ + __clear_bit(nr, addr); +} + +__rust_helper +void rust_helper_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + set_bit(nr, addr); +} + +__rust_helper +void rust_helper_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + clear_bit(nr, addr); +} + +/* + * The rust_helper_ prefix is intentionally omitted below so that the + * declarations in include/linux/find.h are compatible with these helpers. + * + * Note that the below #ifdefs mean that the helper is only created if C does + * not provide a definition. + */ +#ifdef find_first_zero_bit +__rust_helper +unsigned long _find_first_zero_bit(const unsigned long *p, unsigned long size) +{ + return find_first_zero_bit(p, size); +} +#endif /* find_first_zero_bit */ + +#ifdef find_next_zero_bit +__rust_helper +unsigned long _find_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + return find_next_zero_bit(addr, size, offset); +} +#endif /* find_next_zero_bit */ + +#ifdef find_first_bit +__rust_helper +unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) +{ + return find_first_bit(addr, size); +} +#endif /* find_first_bit */ + +#ifdef find_next_bit +__rust_helper +unsigned long _find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + return find_next_bit(addr, size, offset); +} +#endif /* find_next_bit */ diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c new file mode 100644 index 000000000..20c512e46 --- /dev/null +++ b/rust/helpers/blk.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +{ + return blk_mq_rq_to_pdu(rq); +} + +__rust_helper struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) +{ + return blk_mq_rq_from_pdu(pdu); +} diff --git a/rust/helpers/bug.c b/rust/helpers/bug.c new file mode 100644 index 000000000..b51e60772 --- /dev/null +++ b/rust/helpers/bug.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper __noreturn void rust_helper_BUG(void) +{ + BUG(); +} + +__rust_helper bool rust_helper_WARN_ON(bool cond) +{ + return WARN_ON(cond); +} diff --git a/rust/helpers/build_assert.c b/rust/helpers/build_assert.c new file mode 100644 index 000000000..6a54b2680 --- /dev/null +++ b/rust/helpers/build_assert.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can + * use it in contexts where Rust expects a `usize` like slice (array) indices. + * `usize` is defined to be the same as C's `uintptr_t` type (can hold any + * pointer) but not necessarily the same as `size_t` (can hold the size of any + * single object). Most modern platforms use the same concrete integer type for + * both of them, but in case we find ourselves on a platform where + * that's not true, fail early instead of risking ABI or + * integer-overflow issues. + * + * If your platform fails this assertion, it means that you are in + * danger of integer-overflow bugs (even if you attempt to add + * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on + * your platform such that `size_t` matches `uintptr_t` (i.e., to increase + * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). + */ +static_assert( + sizeof(size_t) == sizeof(uintptr_t) && + __alignof__(size_t) == __alignof__(uintptr_t), + "Rust code expects C `size_t` to match Rust `usize`" +); diff --git a/rust/helpers/build_bug.c b/rust/helpers/build_bug.c new file mode 100644 index 000000000..14dbc55bb --- /dev/null +++ b/rust/helpers/build_bug.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper const char *rust_helper_errname(int err) +{ + return errname(err); +} diff --git a/rust/helpers/clk.c b/rust/helpers/clk.c new file mode 100644 index 000000000..15fd7e469 --- /dev/null +++ b/rust/helpers/clk.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * The "inline" implementation of below helpers are only available when + * CONFIG_HAVE_CLK or CONFIG_HAVE_CLK_PREPARE aren't set. + */ +#ifndef CONFIG_HAVE_CLK +__rust_helper struct clk *rust_helper_clk_get(struct device *dev, + const char *id) +{ + return clk_get(dev, id); +} + +__rust_helper void rust_helper_clk_put(struct clk *clk) +{ + clk_put(clk); +} + +__rust_helper int rust_helper_clk_enable(struct clk *clk) +{ + return clk_enable(clk); +} + +__rust_helper void rust_helper_clk_disable(struct clk *clk) +{ + clk_disable(clk); +} + +__rust_helper unsigned long rust_helper_clk_get_rate(struct clk *clk) +{ + return clk_get_rate(clk); +} + +__rust_helper int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate) +{ + return clk_set_rate(clk, rate); +} +#endif + +#ifndef CONFIG_HAVE_CLK_PREPARE +__rust_helper int rust_helper_clk_prepare(struct clk *clk) +{ + return clk_prepare(clk); +} + +__rust_helper void rust_helper_clk_unprepare(struct clk *clk) +{ + clk_unprepare(clk); +} +#endif + +__rust_helper struct clk *rust_helper_clk_get_optional(struct device *dev, + const char *id) +{ + return clk_get_optional(dev, id); +} + +__rust_helper int rust_helper_clk_prepare_enable(struct clk *clk) +{ + return clk_prepare_enable(clk); +} + +__rust_helper void rust_helper_clk_disable_unprepare(struct clk *clk) +{ + clk_disable_unprepare(clk); +} diff --git a/rust/helpers/completion.c b/rust/helpers/completion.c new file mode 100644 index 000000000..0126767cc --- /dev/null +++ b/rust/helpers/completion.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_completion(struct completion *x) +{ + init_completion(x); +} diff --git a/rust/helpers/cpu.c b/rust/helpers/cpu.c new file mode 100644 index 000000000..5759349b2 --- /dev/null +++ b/rust/helpers/cpu.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper unsigned int rust_helper_raw_smp_processor_id(void) +{ + return raw_smp_processor_id(); +} diff --git a/rust/helpers/cpufreq.c b/rust/helpers/cpufreq.c new file mode 100644 index 000000000..0e16aeef2 --- /dev/null +++ b/rust/helpers/cpufreq.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_CPU_FREQ +__rust_helper void +rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy) +{ + cpufreq_register_em_with_opp(policy); +} +#endif diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c new file mode 100644 index 000000000..5deced5b9 --- /dev/null +++ b/rust/helpers/cpumask.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + cpumask_set_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper___cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + __cpumask_set_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + cpumask_clear_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper___cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + __cpumask_clear_cpu(cpu, dstp); +} + +__rust_helper +bool rust_helper_cpumask_test_cpu(int cpu, struct cpumask *srcp) +{ + return cpumask_test_cpu(cpu, srcp); +} + +__rust_helper +void rust_helper_cpumask_setall(struct cpumask *dstp) +{ + cpumask_setall(dstp); +} + +__rust_helper +bool rust_helper_cpumask_empty(struct cpumask *srcp) +{ + return cpumask_empty(srcp); +} + +__rust_helper +bool rust_helper_cpumask_full(struct cpumask *srcp) +{ + return cpumask_full(srcp); +} + +__rust_helper +unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) +{ + return cpumask_weight(srcp); +} + +__rust_helper +void rust_helper_cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) +{ + cpumask_copy(dstp, srcp); +} + +__rust_helper +bool rust_helper_alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var(mask, flags); +} + +__rust_helper +bool rust_helper_zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return zalloc_cpumask_var(mask, flags); +} + +#ifndef CONFIG_CPUMASK_OFFSTACK +__rust_helper +void rust_helper_free_cpumask_var(cpumask_var_t mask) +{ + free_cpumask_var(mask); +} +#endif diff --git a/rust/helpers/cred.c b/rust/helpers/cred.c new file mode 100644 index 000000000..a56a7b753 --- /dev/null +++ b/rust/helpers/cred.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper const struct cred *rust_helper_get_cred(const struct cred *cred) +{ + return get_cred(cred); +} + +__rust_helper void rust_helper_put_cred(const struct cred *cred) +{ + put_cred(cred); +} diff --git a/rust/helpers/device.c b/rust/helpers/device.c new file mode 100644 index 000000000..3be4ee590 --- /dev/null +++ b/rust/helpers/device.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_devm_add_action(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action(dev, action, data); +} + +__rust_helper int rust_helper_devm_add_action_or_reset(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action_or_reset(dev, action, data); +} + +__rust_helper void *rust_helper_dev_get_drvdata(const struct device *dev) +{ + return dev_get_drvdata(dev); +} + +__rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data) +{ + dev_set_drvdata(dev, data); +} + +__rust_helper const char *rust_helper_dev_name(const struct device *dev) +{ + return dev_name(dev); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 000000000..71914d824 --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +__rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c new file mode 100644 index 000000000..9fbeb507b --- /dev/null +++ b/rust/helpers/dma.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, + gfp_t flag, unsigned long attrs) +{ + return dma_alloc_attrs(dev, size, dma_handle, flag, attrs); +} + +__rust_helper void rust_helper_dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, + dma_addr_t dma_handle, + unsigned long attrs) +{ + dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs); +} + +__rust_helper int rust_helper_dma_set_mask_and_coherent(struct device *dev, + u64 mask) +{ + return dma_set_mask_and_coherent(dev, mask); +} + +__rust_helper int rust_helper_dma_set_mask(struct device *dev, u64 mask) +{ + return dma_set_mask(dev, mask); +} + +__rust_helper int rust_helper_dma_set_coherent_mask(struct device *dev, u64 mask) +{ + return dma_set_coherent_mask(dev, mask); +} + +__rust_helper int rust_helper_dma_map_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, unsigned long attrs) +{ + return dma_map_sgtable(dev, sgt, dir, attrs); +} + +__rust_helper size_t rust_helper_dma_max_mapping_size(struct device *dev) +{ + return dma_max_mapping_size(dev); +} + +__rust_helper void rust_helper_dma_set_max_seg_size(struct device *dev, + unsigned int size) +{ + dma_set_max_seg_size(dev, size); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c new file mode 100644 index 000000000..65f3f22b0 --- /dev/null +++ b/rust/helpers/drm.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#ifdef CONFIG_DRM + +__rust_helper void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) +{ + drm_gem_object_get(obj); +} + +__rust_helper void rust_helper_drm_gem_object_put(struct drm_gem_object *obj) +{ + drm_gem_object_put(obj); +} + +__rust_helper __u64 +rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) +{ + return drm_vma_node_offset_addr(node); +} + +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +__rust_helper void +rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +__rust_helper struct sg_table * +rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 000000000..4130b6325 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +#ifdef CONFIG_RUST_DRM_GPUVM + +#include + +__rust_helper +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +__rust_helper +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +__rust_helper +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif // CONFIG_RUST_DRM_GPUVM diff --git a/rust/helpers/err.c b/rust/helpers/err.c new file mode 100644 index 000000000..2872158e3 --- /dev/null +++ b/rust/helpers/err.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper __force void *rust_helper_ERR_PTR(long err) +{ + return ERR_PTR(err); +} + +__rust_helper bool rust_helper_IS_ERR(__force const void *ptr) +{ + return IS_ERR(ptr); +} + +__rust_helper long rust_helper_PTR_ERR(__force const void *ptr) +{ + return PTR_ERR(ptr); +} diff --git a/rust/helpers/fs.c b/rust/helpers/fs.c new file mode 100644 index 000000000..789d60fb8 --- /dev/null +++ b/rust/helpers/fs.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2024 Google LLC. + */ + +#include + +__rust_helper struct file *rust_helper_get_file(struct file *f) +{ + return get_file(f); +} diff --git a/rust/helpers/fwctl.c b/rust/helpers/fwctl.c new file mode 100644 index 000000000..c7eecd433 --- /dev/null +++ b/rust/helpers/fwctl.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#if IS_ENABLED(CONFIG_RUST_FWCTL_ABSTRACTIONS) + +__rust_helper struct fwctl_device *rust_helper_fwctl_get(struct fwctl_device *fwctl) +{ + return fwctl_get(fwctl); +} + +__rust_helper void rust_helper_fwctl_put(struct fwctl_device *fwctl) +{ + fwctl_put(fwctl); +} + +#endif diff --git a/rust/helpers/gpu.c b/rust/helpers/gpu.c new file mode 100644 index 000000000..a25448d54 --- /dev/null +++ b/rust/helpers/gpu.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_GPU_BUDDY + +__rust_helper u64 rust_helper_gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return gpu_buddy_block_offset(block); +} + +__rust_helper unsigned int rust_helper_gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_order(block); +} + +#endif /* CONFIG_GPU_BUDDY */ diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c new file mode 100644 index 000000000..440fb7638 --- /dev/null +++ b/rust/helpers/helpers.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions + * cannot be called either. This file explicitly creates functions ("helpers") + * that wrap those so that they can be called from Rust. + * + * Sorted alphabetically. + */ + +#include + +#ifdef __BINDGEN__ +// Omit `inline` for bindgen as it ignores inline functions. +#define __rust_helper +#else +// The helper functions are all inline functions. +// +// We use `__always_inline` here to bypass LLVM inlining checks, in case the +// helpers are inlined directly into Rust CGUs. +// +// The LLVM inlining checks are false positives: +// * LLVM doesn't want to inline functions compiled with +// `-fno-delete-null-pointer-checks` with code compiled without. +// The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay +// since this is one of the hardening features that does not change the ABI, +// and we shouldn't have null pointer dereferences in these helpers. +// * LLVM doesn't want to inline functions with different list of builtins. C +// side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they +// should be compatible, but LLVM does not perform inlining due to attributes +// mismatch. +// * clang and Rust doesn't have the exact target string. Clang generates +// `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will +// complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always +// enable these features, so they are in fact the same target string, but +// LLVM doesn't understand this and so inlining is inhibited. This can be +// bypassed with `--ignore-tti-inline-compatible`, but this is a hidden +// option. +#define __rust_helper __always_inline +#endif + +#include "atomic.c" +#include "atomic_ext.c" +#include "auxiliary.c" +#include "barrier.c" +#include "binder.c" +#include "bitmap.c" +#include "bitops.c" +#include "blk.c" +#include "bug.c" +#include "build_assert.c" +#include "build_bug.c" +#include "clk.c" +#include "completion.c" +#include "cpu.c" +#include "cpufreq.c" +#include "cpumask.c" +#include "cred.c" +#include "device.c" +#include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER +#include "dma-resv.c" +#endif +#include "drm.c" +#include "drm_gpuvm.c" +#include "err.c" +#include "fs.c" +#include "fwctl.c" +#include "gpu.c" +#include "interrupt.c" +#include "io.c" +#include "irq.c" +#include "jump_label.c" +#include "kunit.c" +#include "list.c" +#include "maple_tree.c" +#include "mm.c" +#include "mutex.c" +#include "net/genetlink.c" +#include "of.c" +#include "page.c" +#include "pci.c" +#include "pid_namespace.c" +#include "platform.c" +#include "poll.c" +#include "processor.c" +#include "property.c" +#include "pwm.c" +#include "rbtree.c" +#include "rcu.c" +#include "refcount.c" +#include "regulator.c" +#include "scatterlist.c" +#include "security.c" +#include "serdev.c" +#include "signal.c" +#include "slab.c" +#include "spinlock.c" +#include "string.c" +#include "srcu.c" +#include "sync.c" +#include "task.c" +#include "time.c" +#include "uaccess.c" +#include "usb.c" +#include "vmalloc.c" +#include "wait.c" +#include "workqueue.c" +#include "xarray.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000..695954986 --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} diff --git a/rust/helpers/io.c b/rust/helpers/io.c new file mode 100644 index 000000000..308950aae --- /dev/null +++ b/rust/helpers/io.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#ifdef CONFIG_HAS_IOMEM +__rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) +{ + return ioremap(offset, size); +} + +__rust_helper void __iomem *rust_helper_ioremap_np(phys_addr_t offset, + size_t size) +{ + return ioremap_np(offset, size); +} + +__rust_helper void rust_helper_iounmap(void __iomem *addr) +{ + iounmap(addr); +} +#endif /* CONFIG_HAS_IOMEM */ + +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} + +__rust_helper u8 rust_helper_readb(const void __iomem *addr) +{ + return readb(addr); +} + +__rust_helper u16 rust_helper_readw(const void __iomem *addr) +{ + return readw(addr); +} + +__rust_helper u32 rust_helper_readl(const void __iomem *addr) +{ + return readl(addr); +} + +#ifdef CONFIG_64BIT +__rust_helper u64 rust_helper_readq(const void __iomem *addr) +{ + return readq(addr); +} +#endif + +__rust_helper void rust_helper_writeb(u8 value, void __iomem *addr) +{ + writeb(value, addr); +} + +__rust_helper void rust_helper_writew(u16 value, void __iomem *addr) +{ + writew(value, addr); +} + +__rust_helper void rust_helper_writel(u32 value, void __iomem *addr) +{ + writel(value, addr); +} + +#ifdef CONFIG_64BIT +__rust_helper void rust_helper_writeq(u64 value, void __iomem *addr) +{ + writeq(value, addr); +} +#endif + +__rust_helper u8 rust_helper_readb_relaxed(const void __iomem *addr) +{ + return readb_relaxed(addr); +} + +__rust_helper u16 rust_helper_readw_relaxed(const void __iomem *addr) +{ + return readw_relaxed(addr); +} + +__rust_helper u32 rust_helper_readl_relaxed(const void __iomem *addr) +{ + return readl_relaxed(addr); +} + +#ifdef CONFIG_64BIT +__rust_helper u64 rust_helper_readq_relaxed(const void __iomem *addr) +{ + return readq_relaxed(addr); +} +#endif + +__rust_helper void rust_helper_writeb_relaxed(u8 value, void __iomem *addr) +{ + writeb_relaxed(value, addr); +} + +__rust_helper void rust_helper_writew_relaxed(u16 value, void __iomem *addr) +{ + writew_relaxed(value, addr); +} + +__rust_helper void rust_helper_writel_relaxed(u32 value, void __iomem *addr) +{ + writel_relaxed(value, addr); +} + +#ifdef CONFIG_64BIT +__rust_helper void rust_helper_writeq_relaxed(u64 value, void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +#endif + +__rust_helper resource_size_t rust_helper_resource_size(struct resource *res) +{ + return resource_size(res); +} + +__rust_helper struct resource * +rust_helper_request_mem_region(resource_size_t start, resource_size_t n, + const char *name) +{ + return request_mem_region(start, n, name); +} + +__rust_helper void rust_helper_release_mem_region(resource_size_t start, + resource_size_t n) +{ + release_mem_region(start, n); +} + +__rust_helper struct resource *rust_helper_request_region(resource_size_t start, + resource_size_t n, + const char *name) +{ + return request_region(start, n, name); +} + +__rust_helper struct resource * +rust_helper_request_muxed_region(resource_size_t start, resource_size_t n, + const char *name) +{ + return request_muxed_region(start, n, name); +} + +__rust_helper void rust_helper_release_region(resource_size_t start, + resource_size_t n) +{ + release_region(start, n); +} diff --git a/rust/helpers/irq.c b/rust/helpers/irq.c new file mode 100644 index 000000000..a61fad333 --- /dev/null +++ b/rust/helpers/irq.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_request_irq(unsigned int irq, + irq_handler_t handler, + unsigned long flags, const char *name, + void *dev) +{ + return request_irq(irq, handler, flags, name, dev); +} diff --git a/rust/helpers/jump_label.c b/rust/helpers/jump_label.c new file mode 100644 index 000000000..7ca384e73 --- /dev/null +++ b/rust/helpers/jump_label.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2024 Google LLC. + */ + +#include + +#ifndef CONFIG_JUMP_LABEL +__rust_helper int rust_helper_static_key_count(struct static_key *key) +{ + return static_key_count(key); +} +#endif diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c new file mode 100644 index 000000000..cafb94b67 --- /dev/null +++ b/rust/helpers/kunit.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper struct kunit *rust_helper_kunit_get_current_test(void) +{ + return kunit_get_current_test(); +} diff --git a/rust/helpers/list.c b/rust/helpers/list.c new file mode 100644 index 000000000..18095a559 --- /dev/null +++ b/rust/helpers/list.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Helpers for C circular doubly linked list implementation. + */ + +#include + +__rust_helper void rust_helper_INIT_LIST_HEAD(struct list_head *list) +{ + INIT_LIST_HEAD(list); +} + +__rust_helper void rust_helper_list_add_tail(struct list_head *new, struct list_head *head) +{ + list_add_tail(new, head); +} diff --git a/rust/helpers/maple_tree.c b/rust/helpers/maple_tree.c new file mode 100644 index 000000000..5586486a7 --- /dev/null +++ b/rust/helpers/maple_tree.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mt_init_flags(struct maple_tree *mt, + unsigned int flags) +{ + mt_init_flags(mt, flags); +} diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c new file mode 100644 index 000000000..b5540997b --- /dev/null +++ b/rust/helpers/mm.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_mmgrab(struct mm_struct *mm) +{ + mmgrab(mm); +} + +__rust_helper void rust_helper_mmdrop(struct mm_struct *mm) +{ + mmdrop(mm); +} + +__rust_helper void rust_helper_mmget(struct mm_struct *mm) +{ + mmget(mm); +} + +__rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) +{ + return mmget_not_zero(mm); +} + +__rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) +{ + mmap_read_lock(mm); +} + +__rust_helper bool rust_helper_mmap_read_trylock(struct mm_struct *mm) +{ + return mmap_read_trylock(mm); +} + +__rust_helper void rust_helper_mmap_read_unlock(struct mm_struct *mm) +{ + mmap_read_unlock(mm); +} + +__rust_helper struct vm_area_struct * +rust_helper_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + return vma_lookup(mm, addr); +} + +__rust_helper void rust_helper_vma_end_read(struct vm_area_struct *vma) +{ + vma_end_read(vma); +} diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c new file mode 100644 index 000000000..1b07d6e64 --- /dev/null +++ b/rust/helpers/mutex.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mutex_lock(struct mutex *lock) +{ + mutex_lock(lock); +} + +__rust_helper int rust_helper_mutex_trylock(struct mutex *lock) +{ + return mutex_trylock(lock); +} + +__rust_helper void rust_helper___mutex_init(struct mutex *mutex, + const char *name, + struct lock_class_key *key) +{ + __mutex_init(mutex, name, key); +} + +__rust_helper void rust_helper_mutex_assert_is_held(struct mutex *mutex) +{ + lockdep_assert_held(mutex); +} + +__rust_helper void rust_helper_mutex_destroy(struct mutex *lock) +{ + mutex_destroy(lock); +} diff --git a/rust/helpers/net/genetlink.c b/rust/helpers/net/genetlink.c new file mode 100644 index 000000000..3530b69f6 --- /dev/null +++ b/rust/helpers/net/genetlink.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2026 Google LLC. + */ + +#include + +#ifdef CONFIG_NET + +__rust_helper struct sk_buff *rust_helper_genlmsg_new(size_t payload, gfp_t flags) +{ + return genlmsg_new(payload, flags); +} + +__rust_helper +int rust_helper_genlmsg_multicast(const struct genl_family *family, + struct sk_buff *skb, u32 portid, + unsigned int group, gfp_t flags) +{ + return genlmsg_multicast(family, skb, portid, group, flags); +} + +__rust_helper void rust_helper_genlmsg_cancel(struct sk_buff *skb, void *hdr) +{ + genlmsg_cancel(skb, hdr); +} + +__rust_helper void rust_helper_genlmsg_end(struct sk_buff *skb, void *hdr) +{ + genlmsg_end(skb, hdr); +} + +__rust_helper void rust_helper_nlmsg_free(struct sk_buff *skb) +{ + nlmsg_free(skb); +} + +__rust_helper +int rust_helper_genl_has_listeners(const struct genl_family *family, + struct net *net, unsigned int group) +{ + return genl_has_listeners(family, net, group); +} + +#endif diff --git a/rust/helpers/of.c b/rust/helpers/of.c new file mode 100644 index 000000000..8f62ca69e --- /dev/null +++ b/rust/helpers/of.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) +{ + return is_of_node(fwnode); +} diff --git a/rust/helpers/page.c b/rust/helpers/page.c new file mode 100644 index 000000000..f8463fbed --- /dev/null +++ b/rust/helpers/page.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +__rust_helper struct page *rust_helper_alloc_pages(gfp_t gfp_mask, + unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} + +__rust_helper void *rust_helper_kmap_local_page(struct page *page) +{ + return kmap_local_page(page); +} + +__rust_helper void rust_helper_kunmap_local(const void *addr) +{ + kunmap_local(addr); +} + +#ifndef NODE_NOT_IN_PAGE_FLAGS +__rust_helper int rust_helper_page_to_nid(const struct page *page) +{ + return page_to_nid(page); +} +#endif diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c new file mode 100644 index 000000000..a714cc2bf --- /dev/null +++ b/rust/helpers/pci.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper u16 rust_helper_pci_dev_id(struct pci_dev *dev) +{ + return PCI_DEVID(dev->bus->number, dev->devfn); +} + +__rust_helper resource_size_t +rust_helper_pci_resource_start(struct pci_dev *pdev, int bar) +{ + return pci_resource_start(pdev, bar); +} + +__rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, + int bar) +{ + return pci_resource_len(pdev, bar); +} + +__rust_helper bool rust_helper_dev_is_pci(const struct device *dev) +{ + return dev_is_pci(dev); +} + +__rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev) +{ + return pci_irq_type(pdev); +} + +#ifndef CONFIG_PCI_IOV +__rust_helper unsigned int +rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev) +{ + return pci_sriov_get_totalvfs(pdev); +} +#endif + +#ifndef CONFIG_PCI_MSI +__rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, + unsigned int min_vecs, + unsigned int max_vecs, + unsigned int flags) +{ + return pci_alloc_irq_vectors(dev, min_vecs, max_vecs, flags); +} + +__rust_helper void rust_helper_pci_free_irq_vectors(struct pci_dev *dev) +{ + pci_free_irq_vectors(dev); +} + +__rust_helper int rust_helper_pci_irq_vector(struct pci_dev *pdev, + unsigned int nvec) +{ + return pci_irq_vector(pdev, nvec); +} +#endif diff --git a/rust/helpers/pid_namespace.c b/rust/helpers/pid_namespace.c new file mode 100644 index 000000000..f46ab779b --- /dev/null +++ b/rust/helpers/pid_namespace.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper struct pid_namespace * +rust_helper_get_pid_ns(struct pid_namespace *ns) +{ + return get_pid_ns(ns); +} + +__rust_helper void rust_helper_put_pid_ns(struct pid_namespace *ns) +{ + put_pid_ns(ns); +} + +/* Get a reference on a task's pid namespace. */ +__rust_helper struct pid_namespace * +rust_helper_task_get_pid_ns(struct task_struct *task) +{ + struct pid_namespace *pid_ns; + + guard(rcu)(); + pid_ns = task_active_pid_ns(task); + if (pid_ns) + get_pid_ns(pid_ns); + return pid_ns; +} diff --git a/rust/helpers/platform.c b/rust/helpers/platform.c new file mode 100644 index 000000000..188b3240f --- /dev/null +++ b/rust/helpers/platform.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_dev_is_platform(const struct device *dev) +{ + return dev_is_platform(dev); +} diff --git a/rust/helpers/poll.c b/rust/helpers/poll.c new file mode 100644 index 000000000..78b3839b5 --- /dev/null +++ b/rust/helpers/poll.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_poll_wait(struct file *filp, + wait_queue_head_t *wait_address, + poll_table *p) +{ + poll_wait(filp, wait_address, p); +} diff --git a/rust/helpers/processor.c b/rust/helpers/processor.c new file mode 100644 index 000000000..76fadbb64 --- /dev/null +++ b/rust/helpers/processor.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_cpu_relax(void) +{ + cpu_relax(); +} diff --git a/rust/helpers/property.c b/rust/helpers/property.c new file mode 100644 index 000000000..8fb990053 --- /dev/null +++ b/rust/helpers/property.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode) +{ + fwnode_handle_put(fwnode); +} diff --git a/rust/helpers/pwm.c b/rust/helpers/pwm.c new file mode 100644 index 000000000..eb24d2ea8 --- /dev/null +++ b/rust/helpers/pwm.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2025 Samsung Electronics Co., Ltd. +// Author: Michal Wilczynski + +#include + +__rust_helper struct device *rust_helper_pwmchip_parent(const struct pwm_chip *chip) +{ + return pwmchip_parent(chip); +} + +__rust_helper void *rust_helper_pwmchip_get_drvdata(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} + +__rust_helper void rust_helper_pwmchip_set_drvdata(struct pwm_chip *chip, void *data) +{ + pwmchip_set_drvdata(chip, data); +} diff --git a/rust/helpers/rbtree.c b/rust/helpers/rbtree.c new file mode 100644 index 000000000..a85defb22 --- /dev/null +++ b/rust/helpers/rbtree.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_rb_link_node(struct rb_node *node, + struct rb_node *parent, + struct rb_node **rb_link) +{ + rb_link_node(node, parent, rb_link); +} + +__rust_helper struct rb_node *rust_helper_rb_first(const struct rb_root *root) +{ + return rb_first(root); +} + +__rust_helper struct rb_node *rust_helper_rb_last(const struct rb_root *root) +{ + return rb_last(root); +} diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c new file mode 100644 index 000000000..481274c05 --- /dev/null +++ b/rust/helpers/rcu.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_rcu_read_lock(void) +{ + rcu_read_lock(); +} + +__rust_helper void rust_helper_rcu_read_unlock(void) +{ + rcu_read_unlock(); +} diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c new file mode 100644 index 000000000..36334a674 --- /dev/null +++ b/rust/helpers/refcount.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper refcount_t rust_helper_REFCOUNT_INIT(int n) +{ + return (refcount_t)REFCOUNT_INIT(n); +} + +__rust_helper void rust_helper_refcount_set(refcount_t *r, int n) +{ + refcount_set(r, n); +} + +__rust_helper void rust_helper_refcount_inc(refcount_t *r) +{ + refcount_inc(r); +} + +__rust_helper void rust_helper_refcount_dec(refcount_t *r) +{ + refcount_dec(r); +} + +__rust_helper bool rust_helper_refcount_dec_and_test(refcount_t *r) +{ + return refcount_dec_and_test(r); +} diff --git a/rust/helpers/regulator.c b/rust/helpers/regulator.c new file mode 100644 index 000000000..9ec5237f4 --- /dev/null +++ b/rust/helpers/regulator.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifndef CONFIG_REGULATOR + +__rust_helper void rust_helper_regulator_put(struct regulator *regulator) +{ + regulator_put(regulator); +} + +__rust_helper int rust_helper_regulator_set_voltage(struct regulator *regulator, + int min_uV, int max_uV) +{ + return regulator_set_voltage(regulator, min_uV, max_uV); +} + +__rust_helper int rust_helper_regulator_get_voltage(struct regulator *regulator) +{ + return regulator_get_voltage(regulator); +} + +__rust_helper struct regulator *rust_helper_regulator_get(struct device *dev, + const char *id) +{ + return regulator_get(dev, id); +} + +__rust_helper int rust_helper_regulator_enable(struct regulator *regulator) +{ + return regulator_enable(regulator); +} + +__rust_helper int rust_helper_regulator_disable(struct regulator *regulator) +{ + return regulator_disable(regulator); +} + +__rust_helper int rust_helper_regulator_is_enabled(struct regulator *regulator) +{ + return regulator_is_enabled(regulator); +} + +__rust_helper int rust_helper_devm_regulator_get_enable(struct device *dev, + const char *id) +{ + return devm_regulator_get_enable(dev, id); +} + +__rust_helper int +rust_helper_devm_regulator_get_enable_optional(struct device *dev, + const char *id) +{ + return devm_regulator_get_enable_optional(dev, id); +} + +#endif diff --git a/rust/helpers/scatterlist.c b/rust/helpers/scatterlist.c new file mode 100644 index 000000000..f3c41ea5e --- /dev/null +++ b/rust/helpers/scatterlist.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper dma_addr_t rust_helper_sg_dma_address(struct scatterlist *sg) +{ + return sg_dma_address(sg); +} + +__rust_helper unsigned int rust_helper_sg_dma_len(struct scatterlist *sg) +{ + return sg_dma_len(sg); +} + +__rust_helper struct scatterlist *rust_helper_sg_next(struct scatterlist *sg) +{ + return sg_next(sg); +} + +__rust_helper void rust_helper_dma_unmap_sgtable(struct device *dev, + struct sg_table *sgt, + enum dma_data_direction dir, + unsigned long attrs) +{ + return dma_unmap_sgtable(dev, sgt, dir, attrs); +} diff --git a/rust/helpers/security.c b/rust/helpers/security.c new file mode 100644 index 000000000..8d0a25fcf --- /dev/null +++ b/rust/helpers/security.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifndef CONFIG_SECURITY +__rust_helper void rust_helper_security_cred_getsecid(const struct cred *c, + u32 *secid) +{ + security_cred_getsecid(c, secid); +} + +__rust_helper int rust_helper_security_secid_to_secctx(u32 secid, + struct lsm_context *cp) +{ + return security_secid_to_secctx(secid, cp); +} + +__rust_helper void rust_helper_security_release_secctx(struct lsm_context *cp) +{ + security_release_secctx(cp); +} + +__rust_helper int +rust_helper_security_binder_set_context_mgr(const struct cred *mgr) +{ + return security_binder_set_context_mgr(mgr); +} + +__rust_helper int +rust_helper_security_binder_transaction(const struct cred *from, + const struct cred *to) +{ + return security_binder_transaction(from, to); +} + +__rust_helper int +rust_helper_security_binder_transfer_binder(const struct cred *from, + const struct cred *to) +{ + return security_binder_transfer_binder(from, to); +} + +__rust_helper int rust_helper_security_binder_transfer_file( + const struct cred *from, const struct cred *to, const struct file *file) +{ + return security_binder_transfer_file(from, to, file); +} +#endif diff --git a/rust/helpers/serdev.c b/rust/helpers/serdev.c new file mode 100644 index 000000000..c52b78ca3 --- /dev/null +++ b/rust/helpers/serdev.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_serdev_device_driver_unregister(struct serdev_device_driver *sdrv) +{ + serdev_device_driver_unregister(sdrv); +} + +__rust_helper +void rust_helper_serdev_device_put(struct serdev_device *serdev) +{ + serdev_device_put(serdev); +} + +__rust_helper +void rust_helper_serdev_device_set_client_ops(struct serdev_device *serdev, + const struct serdev_device_ops *ops) +{ + serdev_device_set_client_ops(serdev, ops); +} diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c new file mode 100644 index 000000000..85111186c --- /dev/null +++ b/rust/helpers/signal.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_signal_pending(struct task_struct *t) +{ + return signal_pending(t); +} diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c new file mode 100644 index 000000000..9279f0824 --- /dev/null +++ b/rust/helpers/slab.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_krealloc_node_align(const void *objp, size_t new_size, unsigned long align, + gfp_t flags, int node) +{ + return krealloc_node_align(objp, new_size, align, flags, node); +} + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_kvrealloc_node_align(const void *p, size_t size, unsigned long align, + gfp_t flags, int node) +{ + return kvrealloc_node_align(p, size, align, flags, node); +} diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c new file mode 100644 index 000000000..d53400c15 --- /dev/null +++ b/rust/helpers/spinlock.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper___spin_lock_init(spinlock_t *lock, + const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_SPINLOCK +# if defined(CONFIG_PREEMPT_RT) + __spin_lock_init(lock, name, key, false); +# else /*!CONFIG_PREEMPT_RT */ + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +# endif /* CONFIG_PREEMPT_RT */ +#else /* !CONFIG_DEBUG_SPINLOCK */ + spin_lock_init(lock); +#endif /* CONFIG_DEBUG_SPINLOCK */ +} + +__rust_helper void rust_helper_spin_lock(spinlock_t *lock) +{ + spin_lock(lock); +} + +__rust_helper void rust_helper_spin_unlock(spinlock_t *lock) +{ + spin_unlock(lock); +} + +__rust_helper int rust_helper_spin_trylock(spinlock_t *lock) +{ + return spin_trylock(lock); +} + +__rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) +{ + lockdep_assert_held(lock); +} + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000..1a2f56364 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/helpers/string.c b/rust/helpers/string.c new file mode 100644 index 000000000..8ef30eb07 --- /dev/null +++ b/rust/helpers/string.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *rust_helper_memchr(const void *s, int c, size_t n) +{ + return memchr(s, c, n); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c new file mode 100644 index 000000000..4f474fe84 --- /dev/null +++ b/rust/helpers/sync.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_lockdep_register_key(struct lock_class_key *k) +{ + lockdep_register_key(k); +} + +__rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) +{ + lockdep_unregister_key(k); +} + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/helpers/task.c b/rust/helpers/task.c new file mode 100644 index 000000000..b46b1433a --- /dev/null +++ b/rust/helpers/task.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_might_resched(void) +{ + might_resched(); +} + +__rust_helper struct task_struct *rust_helper_get_current(void) +{ + return current; +} + +__rust_helper void rust_helper_get_task_struct(struct task_struct *t) +{ + get_task_struct(t); +} + +__rust_helper void rust_helper_put_task_struct(struct task_struct *t) +{ + put_task_struct(t); +} + +__rust_helper kuid_t rust_helper_task_uid(struct task_struct *task) +{ + return task_uid(task); +} + +#ifndef CONFIG_USER_NS +__rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) +{ + return from_kuid(to, uid); +} +#endif /* CONFIG_USER_NS */ + +__rust_helper bool rust_helper_uid_eq(kuid_t left, kuid_t right) +{ + return uid_eq(left, right); +} + +__rust_helper kuid_t rust_helper_current_euid(void) +{ + return current_euid(); +} + +__rust_helper struct user_namespace *rust_helper_current_user_ns(void) +{ + return current_user_ns(); +} + +__rust_helper pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk, + struct pid_namespace *ns) +{ + return task_tgid_nr_ns(tsk, ns); +} diff --git a/rust/helpers/time.c b/rust/helpers/time.c new file mode 100644 index 000000000..32f495970 --- /dev/null +++ b/rust/helpers/time.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +__rust_helper void rust_helper_fsleep(unsigned long usecs) +{ + fsleep(usecs); +} + +__rust_helper ktime_t rust_helper_ktime_get_real(void) +{ + return ktime_get_real(); +} + +__rust_helper ktime_t rust_helper_ktime_get_boottime(void) +{ + return ktime_get_boottime(); +} + +__rust_helper ktime_t rust_helper_ktime_get_clocktai(void) +{ + return ktime_get_clocktai(); +} + +__rust_helper s64 rust_helper_ktime_to_us(const ktime_t kt) +{ + return ktime_to_us(kt); +} + +__rust_helper s64 rust_helper_ktime_to_ms(const ktime_t kt) +{ + return ktime_to_ms(kt); +} + +__rust_helper void rust_helper_udelay(unsigned long usec) +{ + udelay(usec); +} diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c new file mode 100644 index 000000000..6e59cc9c6 --- /dev/null +++ b/rust/helpers/uaccess.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper unsigned long +rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return copy_from_user(to, from, n); +} + +__rust_helper unsigned long +rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return copy_to_user(to, from, n); +} + +#ifdef INLINE_COPY_USER +__rust_helper +unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return _inline_copy_from_user(to, from, n); +} + +__rust_helper +unsigned long rust_helper__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return _inline_copy_to_user(to, from, n); +} +#endif diff --git a/rust/helpers/usb.c b/rust/helpers/usb.c new file mode 100644 index 000000000..eff1cf7be --- /dev/null +++ b/rust/helpers/usb.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper struct usb_device * +rust_helper_interface_to_usbdev(struct usb_interface *intf) +{ + return interface_to_usbdev(intf); +} diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 000000000..6aed13292 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_vrealloc_node_align(const void *p, size_t size, unsigned long align, + gfp_t flags, int node) +{ + return vrealloc_node_align(p, size, align, flags, node); +} + +__rust_helper bool rust_helper_is_vmalloc_addr(const void *x) +{ + return is_vmalloc_addr(x); +} diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c new file mode 100644 index 000000000..2dde1e451 --- /dev/null +++ b/rust/helpers/wait.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +{ + init_wait(wq_entry); +} diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c new file mode 100644 index 000000000..ce1c3a5b2 --- /dev/null +++ b/rust/helpers/workqueue.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_work_with_key(struct work_struct *work, + work_func_t func, + bool onstack, + const char *name, + struct lock_class_key *key) +{ + __init_work(work, onstack); + work->data = (atomic_long_t)WORK_DATA_INIT(); + lockdep_init_map(&work->lockdep_map, name, key, 0); + INIT_LIST_HEAD(&work->entry); + work->func = func; +} diff --git a/rust/helpers/xarray.c b/rust/helpers/xarray.c new file mode 100644 index 000000000..08979b304 --- /dev/null +++ b/rust/helpers/xarray.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_xa_err(void *entry) +{ + return xa_err(entry); +} + +__rust_helper void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) +{ + return xa_init_flags(xa, flags); +} + +__rust_helper int rust_helper_xa_trylock(struct xarray *xa) +{ + return xa_trylock(xa); +} + +__rust_helper void rust_helper_xa_lock(struct xarray *xa) +{ + return xa_lock(xa); +} + +__rust_helper void rust_helper_xa_unlock(struct xarray *xa) +{ + return xa_unlock(xa); +} diff --git a/rust/kernel/.gitignore b/rust/kernel/.gitignore new file mode 100644 index 000000000..f636ad95a --- /dev/null +++ b/rust/kernel/.gitignore @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +/generated_arch_static_branch_asm.rs +/generated_arch_warn_asm.rs +/generated_arch_reachable_asm.rs diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test new file mode 100644 index 000000000..e6a5c7a79 --- /dev/null +++ b/rust/kernel/Kconfig.test @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig RUST_KUNIT_TESTS + bool "Rust KUnit tests" + depends on KUNIT && RUST + default KUNIT_ALL_TESTS + help + This menu collects all options for Rust KUnit tests. + See Documentation/rust/testing.rst for how to protect + unit tests with these options. + + Say Y here to enable Rust KUnit tests. + + If unsure, say N. + +if RUST_KUNIT_TESTS +config RUST_ALLOCATOR_KUNIT_TEST + bool "KUnit tests for Rust allocator API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust allocator API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KVEC_KUNIT_TEST + bool "KUnit tests for Rust KVec API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust KVec API. + These are only for development and testing, not for + regular kernel use cases. + + If unsure, say N. + +config RUST_BITMAP_KUNIT_TEST + bool "KUnit tests for Rust bitmap API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust bitmap API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KUNIT_SELFTEST + bool "KUnit selftests for Rust" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit selftests. These are only + for development and testing, not for regular kernel + use cases. + + If unsure, say N. + +config RUST_STR_KUNIT_TEST + bool "KUnit tests for Rust strings API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust strings API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_ATOMICS_KUNIT_TEST + bool "KUnit tests for Rust atomics API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust atomics API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_BITFIELD_KUNIT_TEST + bool "KUnit tests for the Rust `bitfield!` macro" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust `bitfield!` macro. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +endif diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs new file mode 100644 index 000000000..ea2ce61ee --- /dev/null +++ b/rust/kernel/acpi.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Advanced Configuration and Power Interface abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for ACPI drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// An ACPI device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::acpi_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `acpi_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::acpi_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); +} + +impl DeviceId { + const ACPI_ID_LEN: usize = 16; + + /// Create a new device id from an ACPI 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes"); + let mut acpi: bindings::acpi_device_id = pin_init::zeroed(); + let mut i = 0; + while i < src.len() { + acpi.id[i] = src[i]; + i += 1; + } + + Self(acpi) + } +} + +/// Create an ACPI `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! acpi_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("acpi", $crate::acpi::DeviceId, $($tt)*); + }; +} diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs new file mode 100644 index 000000000..21067bde6 --- /dev/null +++ b/rust/kernel/alloc.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of the kernel's memory allocation infrastructure. + +pub mod allocator; +pub mod kbox; +pub mod kvec; +pub mod layout; + +pub use self::kbox::Box; +pub use self::kbox::KBox; +pub use self::kbox::KVBox; +pub use self::kbox::VBox; + +pub use self::kvec::IntoIter; +pub use self::kvec::KVVec; +pub use self::kvec::KVec; +pub use self::kvec::VVec; +pub use self::kvec::Vec; + +/// Indicates an allocation error. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +use crate::prelude::*; + +use core::{ + alloc::Layout, + ptr::NonNull, // +}; + +/// Flags to be used when allocating memory. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`flags`] module. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(u32); + +impl Flags { + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> u32 { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Allocation flags. +/// +/// These are meant to be used in functions that can allocate memory. +pub mod flags { + use super::Flags; + + /// Zeroes out the allocated memory. + /// + /// This is normally or'd with other flags. + pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO); + + /// Allow the allocation to be in high memory. + /// + /// Allocations in high memory may not be mapped into the kernel's address space, so this can't + /// be used with `kmalloc` and other similar methods. + /// + /// This is normally or'd with other flags. + pub const __GFP_HIGHMEM: Flags = Flags(bindings::__GFP_HIGHMEM); + + /// Users can not sleep and need the allocation to succeed. + /// + /// A lower watermark is applied to allow access to "atomic reserves". The current + /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g. + /// `raw_spin_lock`). The same applies to [`GFP_NOWAIT`]. + pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC); + + /// Typical for kernel-internal allocations. The caller requires `ZONE_NORMAL` or a lower zone + /// for direct access but can direct reclaim. + pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL); + + /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg. + pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT); + + /// For kernel allocations that should not stall for direct reclaim, start physical IO or + /// use any filesystem callback. It is very likely to fail to allocate memory, even for very + /// small allocations. + pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); + + /// Suppresses allocation failure reports. + /// + /// This is normally or'd with other flags. + pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); +} + +/// Non Uniform Memory Access (NUMA) node identifier. +#[derive(Clone, Copy, PartialEq)] +pub struct NumaNode(i32); + +impl NumaNode { + /// Create a new NUMA node identifier (non-negative integer). + /// + /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is + /// specified. + pub fn new(node: i32) -> Result { + // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10. + if node < 0 || node >= bindings::MAX_NUMNODES as i32 { + return Err(EINVAL); + } + Ok(Self(node)) + } +} + +/// Specify necessary constant to pass the information to Allocator that the caller doesn't care +/// about the NUMA node to allocate memory from. +impl NumaNode { + /// No node preference. + pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE); +} + +/// The kernel's [`Allocator`] trait. +/// +/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described +/// via [`Layout`]. +/// +/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on +/// an object instance. +/// +/// In order to be able to support `#[derive(CoercePointee)]` later on, we need to avoid a design +/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind +/// of `self` parameter. +/// +/// # Safety +/// +/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed. +/// +/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`] +/// function of the same type. +/// +/// - Implementers must ensure that all trait functions abide by the guarantees documented in the +/// `# Guarantees` sections. +pub unsafe trait Allocator { + /// The minimum alignment satisfied by all allocations from this allocator. + /// + /// # Guarantees + /// + /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if + /// the requested layout has a smaller alignment. + const MIN_ALIGN: usize; + + /// Allocate memory based on `layout`, `flags` and `nid`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`] or [`Allocator::realloc`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// . + fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result, AllocError> { + // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a + // new memory allocation. + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) } + } + + /// Re-allocate an existing memory allocation to satisfy the requested `layout` and + /// a specific NUMA node request to allocate the memory for. + /// + /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of + /// hardware resources including processors, memory, and I/O buses, that comprise what is + /// commonly known as a NUMA node. + /// + /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer + /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care. + /// + /// If the requested size is zero, `realloc` behaves equivalent to `free`. + /// + /// If the requested size is larger than the size of the existing allocation, a successful call + /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but + /// may also be larger. + /// + /// If the requested size is smaller than the size of the existing allocation, `realloc` may or + /// may not shrink the buffer; this is implementation specific to the allocator. + /// + /// On allocation failure, the existing buffer, if any, remains valid. + /// + /// The buffer is represented as `NonNull<[u8]>`. + /// + /// # Safety + /// + /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation + /// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a + /// pointer returned by this [`Allocator`]. + /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and + /// `old_layout` is ignored. + /// - `old_layout` must match the `Layout` the allocation has been created with. + /// + /// # Guarantees + /// + /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then + /// it additionally guarantees that: + /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new + /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == + /// p[0..min(layout.size(), old_layout.size())]`. + /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError>; + + /// Free an existing memory allocation. + /// + /// # Safety + /// + /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`]; + /// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this + /// [`Allocator`]. + /// - `layout` must match the `Layout` the allocation has been created with. + /// - The memory allocation at `ptr` must never again be read from or written to. + unsafe fn free(ptr: NonNull, layout: Layout) { + // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this + // allocator. We are passing a `Layout` with the smallest possible alignment, so it is + // smaller than or equal to the alignment previously used with this allocation. + let _ = unsafe { + Self::realloc( + Some(ptr), + Layout::new::<()>(), + layout, + Flags(0), + NumaNode::NO_NODE, + ) + }; + } +} + +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs new file mode 100644 index 000000000..cd4203f27 --- /dev/null +++ b/rust/kernel/alloc/allocator.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + bindings, + page, // +}; + +use core::{ + alloc::Layout, + ptr::{ + self, + NonNull, // + }, // +}; + +const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; + +mod iter; +pub use self::iter::VmallocPageIter; + +/// The contiguous kernel allocator. +/// +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. +/// +/// For more details see [self]. +pub struct Kmalloc; + +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; + +/// # Invariants +/// +/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`. +struct ReallocFunc( + unsafe extern "C" fn( + *const crate::ffi::c_void, + usize, + crate::ffi::c_ulong, + u32, + crate::ffi::c_int, + ) -> *mut crate::ffi::c_void, +); + +impl ReallocFunc { + // INVARIANT: `krealloc_node_align` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc_node_align); + + // INVARIANT: `vrealloc_node_align` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc_node_align); + + // INVARIANT: `kvrealloc_node_align` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc_node_align); + + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + #[inline] + unsafe fn call( + &self, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + let size = layout.size(); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; + + // SAFETY: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } +} + +impl Kmalloc { + /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of + /// `layout`. + pub fn aligned_layout(layout: Layout) -> Layout { + // Note that `layout.size()` (after padding) is guaranteed to be a multiple of + // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return + // a properly aligned object (see comments in `kmalloc()` for more information). + layout.pad_to_align() + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + let layout = Kmalloc::aligned_layout(layout); + + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +impl Vmalloc { + /// Convert a pointer to a [`Vmalloc`] allocation to a [`page::BorrowedPage`]. + /// + /// # Examples + /// + /// ``` + /// # use core::ptr::{ + /// # from_mut, + /// # NonNull, // + /// # }; + /// # use kernel::page; + /// use kernel::alloc::allocator::Vmalloc; + /// + /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; + /// + /// { + /// // SAFETY: By the type invariant of `Box` the inner pointer of `vbox` is non-null. + /// let ptr = unsafe { NonNull::new_unchecked(from_mut(&mut *vbox)) }; + /// + /// // SAFETY: + /// // `ptr` is a valid pointer to a `Vmalloc` allocation. + /// // `ptr` is valid for the entire lifetime of `page`. + /// let page = unsafe { Vmalloc::to_page(ptr.cast()) }; + /// + /// // SAFETY: There is no concurrent read or write to the same page. + /// unsafe { page.fill_zero_raw(0, page::PAGE_SIZE)? }; + /// } + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a [`Vmalloc`] allocation. + /// - `ptr` must remain valid for the entire duration of `'a`. + pub unsafe fn to_page<'a>(ptr: NonNull) -> page::BorrowedPage<'a> { + // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. + let page = unsafe { bindings::vmalloc_to_page(ptr.as_ptr().cast()) }; + + // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid pointer + // to `Vmalloc` memory. + let page = unsafe { NonNull::new_unchecked(page) }; + + // SAFETY: + // - `page` is a valid pointer to a `struct page`, given that by the safety requirements of + // this function `ptr` is a valid pointer to a `Vmalloc` allocation. + // - By the safety requirements of this function `ptr` is valid for the entire lifetime of + // `'a`. + unsafe { page::BorrowedPage::from_raw(page) } + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc` + // compatible layout. + let layout = Kmalloc::aligned_layout(layout); + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)] +#[macros::kunit_tests(rust_allocator)] +mod tests { + use super::*; + use core::mem::MaybeUninit; + use kernel::prelude::*; + + #[test] + fn test_alignment() -> Result { + const TEST_SIZE: usize = 1024; + const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4; + + // These two structs are used to test allocating aligned memory. + // they don't need to be accessed, so they're marked as dead_code. + #[expect(dead_code)] + #[repr(align(128))] + struct Blob([u8; TEST_SIZE]); + #[expect(dead_code)] + #[repr(align(8192))] + struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]); + + struct TestAlign(Box, A>); + impl TestAlign { + fn new() -> Result { + Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?)) + } + + fn is_aligned_to(&self, align: usize) -> bool { + assert!(align.is_power_of_two()); + + let addr = self.0.as_ptr() as usize; + addr & (align - 1) == 0 + } + } + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + Ok(()) + } +} diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs new file mode 100644 index 000000000..02fda3ea5 --- /dev/null +++ b/rust/kernel/alloc/allocator/iter.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::Vmalloc; + +use crate::page; + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; + +/// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation. +/// +/// # Guarantees +/// +/// The pages iterated by the [`Iterator`] appear in the order as they are mapped in the CPU's +/// virtual address space ascendingly. +/// +/// # Invariants +/// +/// - `buf` is a valid and [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. +/// - `size` is the number of bytes from `buf` until the end of the [`Vmalloc`] allocation `buf` +/// points to. +pub struct VmallocPageIter<'a> { + /// The base address of the [`Vmalloc`] buffer. + buf: NonNull, + /// The size of the buffer pointed to by `buf` in bytes. + size: usize, + /// The current page index of the [`Iterator`]. + index: usize, + _p: PhantomData>, +} + +impl<'a> Iterator for VmallocPageIter<'a> { + type Item = page::BorrowedPage<'a>; + + fn next(&mut self) -> Option { + let offset = self.index.checked_mul(page::PAGE_SIZE)?; + + // Even though `self.size()` may be smaller than `Self::page_count() * page::PAGE_SIZE`, it + // is always a number between `(Self::page_count() - 1) * page::PAGE_SIZE` and + // `Self::page_count() * page::PAGE_SIZE`, hence the check below is sufficient. + if offset < self.size() { + self.index += 1; + } else { + return None; + } + + // SAFETY: `offset` is in the interval `[0, (self.page_count() - 1) * page::PAGE_SIZE]`, + // hence the resulting pointer is guaranteed to be within the same allocation. + let ptr = unsafe { self.buf.add(offset) }; + + // SAFETY: + // - `ptr` is a valid pointer to a `Vmalloc` allocation. + // - `ptr` is valid for the duration of `'a`. + Some(unsafe { Vmalloc::to_page(ptr) }) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.page_count().saturating_sub(self.index); + + (remaining, Some(remaining)) + } +} + +impl<'a> VmallocPageIter<'a> { + /// Creates a new [`VmallocPageIter`] instance. + /// + /// # Safety + /// + /// - `buf` must be a [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. + /// - `buf` must be valid for at least the lifetime of `'a`. + /// - `size` must be the number of bytes from `buf` until the end of the [`Vmalloc`] allocation + /// `buf` points to. + pub unsafe fn new(buf: NonNull, size: usize) -> Self { + // INVARIANT: By the safety requirements, `buf` is a valid and `page::PAGE_SIZE` aligned + // pointer into a [`Vmalloc`] allocation. + Self { + buf, + size, + index: 0, + _p: PhantomData, + } + } + + /// Returns the size of the backing [`Vmalloc`] allocation in bytes. + /// + /// Note that this is the size the [`Vmalloc`] allocation has been allocated with. Hence, this + /// number may be smaller than `[`Self::page_count`] * [`page::PAGE_SIZE`]`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Returns the number of pages owned by the backing [`Vmalloc`] allocation. + #[inline] + pub fn page_count(&self) -> usize { + self.size().div_ceil(page::PAGE_SIZE) + } +} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs new file mode 100644 index 000000000..c63d6acdb --- /dev/null +++ b/rust/kernel/alloc/kbox.rs @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Box`]. + +#[allow(unused_imports)] // Used in doc comments. +use super::allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // +}; + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::AsPageIter, + prelude::*, + types::ForeignOwnable, // +}; + +use core::{ + alloc::Layout, + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, + result::Result, // +}; + +use pin_init::ZeroableOption; + +/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. +/// +/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences, +/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not +/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`] +/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions +/// that may allocate memory are fallible. +/// +/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]. +/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]). +/// +/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KBox::::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err()); +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KVBox::::new_uninit(GFP_KERNEL).is_ok()); +/// ``` +/// +/// [`Box`]es can also be used to store trait objects by coercing their type: +/// +/// ``` +/// trait FooTrait {} +/// +/// struct FooStruct; +/// impl FooTrait for FooStruct {} +/// +/// let _ = KBox::new(FooStruct, GFP_KERNEL)? as KBox; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct Box<#[pointee] T: ?Sized, A: Allocator>(NonNull, PhantomData); + +/// Type alias for [`Box`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KBox = Box; + +/// Type alias for [`Box`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = VBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type VBox = Box; + +/// Type alias for [`Box`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KVBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KVBox = Box; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// ). +unsafe impl ZeroableOption for Box {} + +// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. +unsafe impl Send for Box +where + T: Send + ?Sized, + A: Allocator, +{ +} + +// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`. +unsafe impl Sync for Box +where + T: Sync + ?Sized, + A: Allocator, +{ +} + +impl Box +where + T: ?Sized, + A: Allocator, +{ + /// Creates a new `Box` from a raw pointer. + /// + /// # Safety + /// + /// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently + /// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the + /// `Box`. + /// + /// For ZSTs, `raw` must be a dangling, well aligned pointer. + #[inline] + pub const unsafe fn from_raw(raw: *mut T) -> Self { + // INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function. + // SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer. + Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData) + } + + /// Consumes the `Box` and returns a raw pointer. + /// + /// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive + /// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the + /// allocation, if any. + /// + /// # Examples + /// + /// ``` + /// let x = KBox::new(24, GFP_KERNEL)?; + /// let ptr = KBox::into_raw(x); + /// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`. + /// let x = unsafe { KBox::from_raw(ptr) }; + /// + /// assert_eq!(*x, 24); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn into_raw(b: Self) -> *mut T { + ManuallyDrop::new(b).0.as_ptr() + } + + /// Consumes and leaks the `Box` and returns a mutable reference. + /// + /// See [`Box::into_raw`] for more details. + #[inline] + pub fn leak<'a>(b: Self) -> &'a mut T { + // SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer + // which points to an initialized instance of `T`. + unsafe { &mut *Box::into_raw(b) } + } +} + +impl Box, A> +where + A: Allocator, +{ + /// Converts a `Box, A>` to a `Box`. + /// + /// It is undefined behavior to call this function while the value inside of `b` is not yet + /// fully initialized. + /// + /// # Safety + /// + /// Callers must ensure that the value inside of `b` is in an initialized state. + pub unsafe fn assume_init(self) -> Box { + let raw = Self::into_raw(self); + + // SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements + // of this function, the value inside the `Box` is in an initialized state. Hence, it is + // safe to reconstruct the `Box` as `Box`. + unsafe { Box::from_raw(raw.cast()) } + } + + /// Writes the value and converts to `Box`. + pub fn write(mut self, value: T) -> Box { + (*self).write(value); + + // SAFETY: We've just initialized `b`'s value. + unsafe { self.assume_init() } + } +} + +impl Box +where + A: Allocator, +{ + /// Creates a new `Box` and initializes its contents with `x`. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + pub fn new(x: T, flags: Flags) -> Result { + let b = Self::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + /// Creates a new `Box` with uninitialized contents. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::::new_uninit(GFP_KERNEL)?; + /// let b = KBox::write(b, 24); + /// + /// assert_eq!(*b, 24_u64); + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { + let layout = Layout::new::>(); + let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?; + + // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, + // which is sufficient in size and alignment for storing a `T`. + Ok(Box(ptr.cast(), PhantomData)) + } + + /// Creates a new zero-initialized `Box`. + /// + /// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in + /// which case an error is returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?; + /// assert_eq!(*b, [0; 128]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(flags: Flags) -> Result + where + T: Zeroable, + { + // SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that + // all-zeroes is a valid bit pattern for `T`. + Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() }) + } + + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then `x` will be + /// pinned in memory and can't be moved. + #[inline] + pub fn pin(x: T, flags: Flags) -> Result>, AllocError> + where + A: 'static, + { + Ok(Self::new(x, flags)?.into()) + } + + /// Construct a pinned slice of elements `Pin>`. + /// + /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks + /// or mutexes. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{ + /// new_spinlock, + /// SpinLock, // + /// }; + /// + /// struct Inner { + /// a: u32, + /// b: u32, + /// } + /// + /// #[pin_data] + /// struct Example { + /// c: u32, + /// #[pin] + /// d: SpinLock, + /// } + /// + /// impl Example { + /// fn new() -> impl PinInit { + /// try_pin_init!(Self { + /// c: 10, + /// d <- new_spinlock!(Inner { a: 20, b: 30 }), + /// }) + /// } + /// } + /// + /// // Allocate a boxed slice of 10 `Example`s. + /// let s = KBox::pin_slice( + /// | _i | Example::new(), + /// 10, + /// GFP_KERNEL + /// )?; + /// + /// assert_eq!(s[5].c, 10); + /// assert_eq!(s[3].d.lock().a, 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pin_slice( + mut init: Func, + len: usize, + flags: Flags, + ) -> Result>, E> + where + Func: FnMut(usize) -> Item, + Item: PinInit, + E: From, + { + let mut buffer = super::Vec::::with_capacity(len, flags)?; + for i in 0..len { + let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast(); + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory. + // - `ptr` is not used if an error is returned. + // - `ptr` won't be moved until it is dropped, i.e. it is pinned. + unsafe { pin_init::raw_try_init(ptr, init(i))? }; + + // SAFETY: + // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to + // `with_capacity()` above. + // - The new value at index buffer.len() + 1 is the only element being added here, and + // it has been initialized above by `raw_try_init(ptr, i)`. + unsafe { buffer.inc_len(1) }; + } + + let (ptr, _, _) = buffer.into_raw_parts(); + let slice = core::ptr::slice_from_raw_parts_mut(ptr, len); + + // SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid + // `[T]`. + Ok(Pin::from(unsafe { Box::from_raw(slice) })) + } + + /// Convert a [`Box`] to a [`Pin>`]. If `T` does not implement + /// [`Unpin`], then `x` will be pinned in memory and can't be moved. + pub fn into_pin(this: Self) -> Pin { + this.into() + } + + /// Forgets the contents (does not run the destructor), but keeps the allocation. + fn forget_contents(this: Self) -> Box, A> { + let ptr = Self::into_raw(this); + + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// let value = KBox::new([0; 32], GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = KBox::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = KBox::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drop_contents(this: Self) -> Box, A> { + let ptr = this.0.as_ptr(); + + // SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the + // value stored in `this` again. + unsafe { core::ptr::drop_in_place(ptr) }; + + Self::forget_contents(this) + } + + /// Moves the `Box`'s value out of the `Box` and consumes the `Box`. + pub fn into_inner(b: Self) -> T { + // SAFETY: By the type invariant `&*b` is valid for `read`. + let value = unsafe { core::ptr::read(&*b) }; + let _ = Self::forget_contents(b); + value + } +} + +impl From> for Pin> +where + T: ?Sized, + A: Allocator, +{ + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*b` will be pinned in memory and can't be moved. + /// + /// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory. + fn from(b: Box) -> Self { + // SAFETY: The value wrapped inside a `Pin>` cannot be moved or replaced as long + // as `T` does not implement `Unpin`. + unsafe { Pin::new_unchecked(b) } + } +} + +impl InPlaceWrite for Box, A> +where + A: Allocator + 'static, +{ + type Initialized = Box; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }.into()) + } +} + +impl InPlaceInit for Box +where + A: Allocator + 'static, +{ + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_init(init) + } +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T` allocated by `A`. +unsafe impl ForeignOwnable for Box +where + A: Allocator, +{ + const FOREIGN_ALIGN: usize = if core::mem::align_of::() < A::MIN_ALIGN { + A::MIN_ALIGN + } else { + core::mem::align_of::() + }; + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a mut T + where + Self: 'a; + + fn into_foreign(self) -> *mut c_void { + Box::into_raw(self).cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Box::from_raw(ptr.cast()) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a mut T { + let ptr = ptr.cast(); + // SAFETY: The safety requirements of this method ensure that the pointer is valid and that + // nothing else will access the value for the duration of 'a. + unsafe { &mut *ptr } + } +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T` allocated by `A`. +unsafe impl ForeignOwnable for Pin> +where + A: Allocator, +{ + const FOREIGN_ALIGN: usize = as ForeignOwnable>::FOREIGN_ALIGN; + type Borrowed<'a> + = Pin<&'a T> + where + Self: 'a; + type BorrowedMut<'a> + = Pin<&'a mut T> + where + Self: 'a; + + fn into_foreign(self) -> *mut c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Pin<&'a mut T> { + let ptr = ptr.cast(); + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &mut *ptr }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } +} + +impl Deref for Box +where + T: ?Sized, + A: Allocator, +{ + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_mut() } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for Box +where + T: ?Sized, + A: Allocator, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +impl fmt::Display for Box +where + T: ?Sized + fmt::Display, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&**self, f) + } +} + +impl fmt::Debug for Box +where + T: ?Sized + fmt::Debug, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&**self, f) + } +} + +impl Drop for Box +where + T: ?Sized, + A: Allocator, +{ + fn drop(&mut self) { + let layout = Layout::for_value::(self); + + // SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant. + unsafe { core::ptr::drop_in_place::(self.deref_mut()) }; + + // SAFETY: + // - `self.0` was previously allocated with `A`. + // - `layout` is equal to the `Layout´ `self.0` was allocated with. + unsafe { A::free(self.0.cast(), layout) }; + } +} + +/// # Examples +/// +/// ``` +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; +/// +/// let mut vbox = VBox::new((), GFP_KERNEL)?; +/// +/// assert!(vbox.page_iter().next().is_none()); +/// +/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; +/// +/// let page = vbox.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VBox { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.0.cast(); + let size = core::mem::size_of::(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs new file mode 100644 index 000000000..c7546b9da --- /dev/null +++ b/rust/kernel/alloc/kvec.rs @@ -0,0 +1,1688 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Vec`]. + +use super::{ + allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // + }, + flags::__GFP_ZERO, + layout::ArrayLayout, + AllocError, + Allocator, + Box, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::{ + AsPageIter, + PAGE_SIZE, // + }, // +}; + +use core::{ + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Index, + IndexMut, // + }, + ptr::{ + self, + NonNull, // + }, + slice::{ + self, + SliceIndex, // + }, // +}; + +use pin_init::Zeroable; + +mod errors; +pub use self::errors::{InsertError, PushError, RemoveError}; + +/// Create a [`KVec`] containing the arguments. +/// +/// New memory is allocated with `GFP_KERNEL`. +/// +/// # Examples +/// +/// ``` +/// let mut v = kernel::kvec![]; +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(v, [1]); +/// +/// let mut v = kernel::kvec![1; 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 1, 1, 4]); +/// +/// let mut v = kernel::kvec![1, 2, 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 2, 3, 4]); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[macro_export] +macro_rules! kvec { + () => ( + $crate::alloc::KVec::new() + ); + ($elem:expr; $n:expr) => ( + $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL) + ); + ($($x:expr),+ $(,)?) => ( + match $crate::alloc::KBox::new_uninit(GFP_KERNEL) { + Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))), + Err(e) => Err(e), + } + ); +} + +/// The kernel's [`Vec`] type. +/// +/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g. +/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec`. +/// +/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For +/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist. +/// +/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::`; no memory is allocated. +/// +/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the +/// capacity of the vector (the number of elements that currently fit into the vector), its length +/// (the number of elements that are currently stored in the vector) and the `Allocator` type used +/// to allocate (and free) the backing buffer. +/// +/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts +/// and manually modified. +/// +/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements +/// are added to the vector. +/// +/// # Invariants +/// +/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +/// +/// - `self.len` always represents the exact number of elements stored in the vector. +/// +/// - `self.layout` represents the absolute number of elements that can be stored within the vector +/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the +/// backing buffer to be larger than `layout`. +/// +/// - `self.len()` is always less than or equal to `self.capacity()`. +/// +/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer +/// was allocated with (and must be freed with). +pub struct Vec { + ptr: NonNull, + /// Represents the actual buffer size as `cap` times `size_of::` bytes. + /// + /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of + /// elements we can still store without reallocating. + layout: ArrayLayout, + len: usize, + _p: PhantomData, +} + +/// Type alias for [`Vec`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVec = Vec; + +/// Type alias for [`Vec`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = VVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type VVec = Vec; + +/// Type alias for [`Vec`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVVec = Vec; + +// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements. +unsafe impl Send for Vec +where + T: Send, + A: Allocator, +{ +} + +// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements. +unsafe impl Sync for Vec +where + T: Sync, + A: Allocator, +{ +} + +impl Vec +where + A: Allocator, +{ + #[inline] + const fn is_zst() -> bool { + core::mem::size_of::() == 0 + } + + /// Returns the number of elements that can be stored within the vector without allocating + /// additional memory. + pub const fn capacity(&self) -> usize { + if const { Self::is_zst() } { + usize::MAX + } else { + self.layout.len() + } + } + + /// Returns the number of elements stored within the vector. + #[inline] + pub const fn len(&self) -> usize { + self.len + } + + /// Increments `self.len` by `additional`. + /// + /// # Safety + /// + /// - `additional` must be less than or equal to `self.capacity - self.len`. + /// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized. + #[inline] + pub const unsafe fn inc_len(&mut self, additional: usize) { + // Guaranteed by the type invariant to never underflow. + debug_assert!(additional <= self.capacity() - self.len()); + // INVARIANT: By the safety requirements of this method this represents the exact number of + // elements stored within `self`. + self.len += additional; + } + + /// Decreases `self.len` by `count`. + /// + /// Returns a mutable slice to the elements forgotten by the vector. It is the caller's + /// responsibility to drop these elements if necessary. + /// + /// # Safety + /// + /// - `count` must be less than or equal to `self.len`. + unsafe fn dec_len(&mut self, count: usize) -> &mut [T] { + debug_assert!(count <= self.len()); + // INVARIANT: We relinquish ownership of the elements within the range `[self.len - count, + // self.len)`, hence the updated value of `set.len` represents the exact number of elements + // stored within `self`. + self.len -= count; + // SAFETY: The memory after `self.len()` is guaranteed to contain `count` initialized + // elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr().add(self.len), count) } + } + + /// Returns a slice of the entire vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(v.as_slice(), &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a + /// dangling raw pointer. + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self.ptr.as_ptr() + } + + /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw + /// pointer. + #[inline] + pub const fn as_ptr(&self) -> *const T { + self.ptr.as_ptr() + } + + /// Returns `true` if the vector contains no elements, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1, GFP_KERNEL); + /// assert!(!v.is_empty()); + /// ``` + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a new, empty `Vec`. + /// + /// This method does not allocate by itself. + #[inline] + pub const fn new() -> Self { + // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet, + // - `ptr` is a properly aligned dangling pointer for type `T`, + // - `layout` is an empty `ArrayLayout` (zero capacity) + // - `len` is zero, since no elements can be or have been stored, + // - `A` is always valid. + Self { + ptr: NonNull::dangling(), + layout: ArrayLayout::empty(), + len: 0, + _p: PhantomData::, + } + } + + /// Returns a slice of `MaybeUninit` for the remaining spare capacity of the vector. + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // SAFETY: + // - `self.len` is smaller than `self.capacity` by the type invariant and hence, the + // resulting pointer is guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) }.cast::>(); + + // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated + // and valid, but uninitialized. + unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) } + } + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + self.reserve(1, flags)?; + // SAFETY: The call to `reserve` was successful, so the capacity is at least one greater + // than the length. + unsafe { self.push_within_capacity_unchecked(v) }; + Ok(()) + } + + /// Appends an element to the back of the [`Vec`] instance without reallocating. + /// + /// Fails if the vector does not have capacity for the new element. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::with_capacity(10, GFP_KERNEL)?; + /// for i in 0..10 { + /// v.push_within_capacity(i)?; + /// } + /// + /// assert!(v.push_within_capacity(10).is_err()); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push_within_capacity(&mut self, v: T) -> Result<(), PushError> { + if self.len() < self.capacity() { + // SAFETY: The length is less than the capacity. + unsafe { self.push_within_capacity_unchecked(v) }; + Ok(()) + } else { + Err(PushError(v)) + } + } + + /// Appends an element to the back of the [`Vec`] instance without reallocating. + /// + /// # Safety + /// + /// The length must be less than the capacity. + unsafe fn push_within_capacity_unchecked(&mut self, v: T) { + let spare = self.spare_capacity_mut(); + + // SAFETY: By the safety requirements, `spare` is non-empty. + unsafe { spare.get_unchecked_mut(0) }.write(v); + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because the caller guarantees that + // the length is less than the capacity at the beginning of this function. + unsafe { self.inc_len(1) }; + } + + /// Inserts an element at the given index in the [`Vec`] instance. + /// + /// Fails if the vector does not have capacity for the new element. Panics if the index is out + /// of bounds. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::kvec::InsertError; + /// + /// let mut v = KVec::with_capacity(5, GFP_KERNEL)?; + /// for i in 0..5 { + /// v.insert_within_capacity(0, i)?; + /// } + /// + /// assert!(matches!(v.insert_within_capacity(0, 5), Err(InsertError::OutOfCapacity(_)))); + /// assert!(matches!(v.insert_within_capacity(1000, 5), Err(InsertError::IndexOutOfBounds(_)))); + /// assert_eq!(v, [4, 3, 2, 1, 0]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn insert_within_capacity( + &mut self, + index: usize, + element: T, + ) -> Result<(), InsertError> { + let len = self.len(); + if index > len { + return Err(InsertError::IndexOutOfBounds(element)); + } + + if len >= self.capacity() { + return Err(InsertError::OutOfCapacity(element)); + } + + // SAFETY: This is in bounds since `index <= len < capacity`. + let p = unsafe { self.as_mut_ptr().add(index) }; + // INVARIANT: This breaks the Vec invariants by making `index` contain an invalid element, + // but we restore the invariants below. + // SAFETY: Both the src and dst ranges end no later than one element after the length. + // Since the length is less than the capacity, both ranges are in bounds of the allocation. + unsafe { ptr::copy(p, p.add(1), len - index) }; + // INVARIANT: This restores the Vec invariants. + // SAFETY: The pointer is in-bounds of the allocation. + unsafe { ptr::write(p, element) }; + // SAFETY: Index `len` contains a valid element due to the above copy and write. + unsafe { self.inc_len(1) }; + Ok(()) + } + + /// Removes the last element from a vector and returns it, or `None` if it is empty. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// + /// assert_eq!(v.pop(), Some(2)); + /// assert_eq!(v.pop(), Some(1)); + /// assert_eq!(v.pop(), None); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pop(&mut self) -> Option { + if self.is_empty() { + return None; + } + + let removed: *mut T = { + // SAFETY: We just checked that the length is at least one. + let slice = unsafe { self.dec_len(1) }; + // SAFETY: The argument to `dec_len` was 1 so this returns a slice of length 1. + unsafe { slice.get_unchecked_mut(0) } + }; + + // SAFETY: The guarantees of `dec_len` allow us to take ownership of this value. + Some(unsafe { removed.read() }) + } + + /// Removes the element at the given index. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// assert_eq!(v.remove(1)?, 2); + /// assert_eq!(v, [1, 3]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn remove(&mut self, i: usize) -> Result { + let value = { + let value_ref = self.get(i).ok_or(RemoveError)?; + // INVARIANT: This breaks the invariants by invalidating the value at index `i`, but we + // restore the invariants below. + // SAFETY: The value at index `i` is valid, because otherwise we would have already + // failed with `RemoveError`. + unsafe { ptr::read(value_ref) } + }; + + // SAFETY: We checked that `i` is in-bounds. + let p = unsafe { self.as_mut_ptr().add(i) }; + + // INVARIANT: After this call, the invalid value is at the last slot, so the Vec invariants + // are restored after the below call to `dec_len(1)`. + // SAFETY: `p.add(1).add(self.len - i - 1)` is `i+1+len-i-1 == len` elements after the + // beginning of the vector, so this is in-bounds of the vector's allocation. + unsafe { ptr::copy(p.add(1), p, self.len - i - 1) }; + + // SAFETY: Since the check at the beginning of this call did not fail with `RemoveError`, + // the length is at least one. + unsafe { self.dec_len(1) }; + + Ok(value) + } + + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn with_capacity(capacity: usize, flags: Flags) -> Result { + let mut v = Vec::new(); + + v.reserve(capacity, flags)?; + + Ok(v) + } + + /// Creates a new [`Vec`] with `n` zero-initialized elements. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::zeroed(20, GFP_KERNEL)?; + /// + /// assert!(v.iter().all(|&x| x == 0)); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(n: usize, flags: Flags) -> Result + where + T: Zeroable, + { + let mut v = Self::with_capacity(n, flags | __GFP_ZERO)?; + + // SAFETY: + // - `n <= capacity - len`: `with_capacity(n)` guarantees capacity >= n, len is 0. + // - All elements in `[0, n)` are initialized: `__GFP_ZERO` zeroes the allocation, + // and `T: Zeroable` guarantees all-zeroes is a valid bit pattern. + unsafe { v.inc_len(n) }; + Ok(v) + } + + /// Creates a `Vec` from a pointer, a length and a capacity using the allocator `A`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.reserve(1, GFP_KERNEL)?; + /// + /// let (mut ptr, mut len, cap) = v.into_raw_parts(); + /// + /// // SAFETY: We've just reserved memory for another element. + /// unsafe { ptr.add(len).write(4) }; + /// len += 1; + /// + /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and + /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it + /// // from the exact same raw parts. + /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) }; + /// + /// assert_eq!(v, [1, 2, 3, 4]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// If `T` is a ZST: + /// + /// - `ptr` must be a dangling, well aligned pointer. + /// + /// Otherwise: + /// + /// - `ptr` must have been allocated with the allocator `A`. + /// - `ptr` must satisfy or exceed the alignment requirements of `T`. + /// - `ptr` must point to memory with a size of at least `size_of::() * capacity` bytes. + /// - The allocated size in bytes must not be larger than `isize::MAX`. + /// - `length` must be less than or equal to `capacity`. + /// - The first `length` elements must be initialized values of type `T`. + /// + /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for + /// `cap` and `len`. + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + let layout = if Self::is_zst() { + ArrayLayout::empty() + } else { + // SAFETY: By the safety requirements of this function, `capacity * size_of::()` is + // smaller than `isize::MAX`. + unsafe { ArrayLayout::new_unchecked(capacity) } + }; + + // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are + // covered by the safety requirements of this function. + Self { + // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid + // memory allocation, allocated with `A`. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + layout, + len: length, + _p: PhantomData::, + } + } + + /// Consumes the `Vec` and returns its raw components `pointer`, `length` and `capacity`. + /// + /// This will not run the destructor of the contained elements and for non-ZSTs the allocation + /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the + /// elements and free the allocation, if any. + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + (ptr, len, capacity) + } + + /// Clears the vector, removing all values. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// + /// v.clear(); + /// + /// assert!(v.is_empty()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn clear(&mut self) { + self.truncate(0); + } + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= v.len() + 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if Self::is_zst() { + // The capacity is already `usize::MAX` for ZSTs, we can't go higher. + return Err(AllocError); + } + + // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the + // multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `ptr` is valid because it's either `None` or comes from a previous call to + // `A::realloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + A::realloc( + Some(self.ptr.cast()), + layout.into(), + self.layout.into(), + flags, + NumaNode::NO_NODE, + )? + }; + + // INVARIANT: + // - `layout` is some `ArrayLayout::`, + // - `ptr` has been created by `A::realloc` from `layout`. + self.ptr = ptr.cast(); + self.layout = layout; + + Ok(()) + } + + /// Shortens the vector, setting the length to `len` and drops the removed values. + /// If `len` is greater than or equal to the current length, this does nothing. + /// + /// This has no effect on the capacity and will not allocate. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.truncate(1); + /// assert_eq!(v.len(), 1); + /// assert_eq!(&v, &[1]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn truncate(&mut self, len: usize) { + if let Some(count) = self.len().checked_sub(len) { + // SAFETY: `count` is `self.len() - len` so it is guaranteed to be less than or + // equal to `self.len()`. + let ptr: *mut [T] = unsafe { self.dec_len(count) }; + + // SAFETY: the contract of `dec_len` guarantees that the elements in `ptr` are + // valid elements whose ownership has been transferred to the caller. + unsafe { ptr::drop_in_place(ptr) }; + } + } + + /// Takes ownership of all items in this vector without consuming the allocation. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![0, 1, 2, 3]?; + /// + /// for (i, j) in v.drain_all().enumerate() { + /// assert_eq!(i, j); + /// } + /// + /// assert!(v.capacity() >= 4); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drain_all(&mut self) -> DrainAll<'_, T> { + // SAFETY: This does not underflow the length. + let elems = unsafe { self.dec_len(self.len()) }; + // INVARIANT: The first `len` elements of the spare capacity are valid values, and as we + // just set the length to zero, we may transfer ownership to the `DrainAll` object. + DrainAll { + elements: elems.iter_mut(), + } + } + + /// Removes all elements that don't match the provided closure. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3, 4]?; + /// v.retain(|i| *i % 2 == 0); + /// assert_eq!(v, [2, 4]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn retain(&mut self, mut f: impl FnMut(&mut T) -> bool) { + let mut num_kept = 0; + let mut next_to_check = 0; + while let Some(to_check) = self.get_mut(next_to_check) { + if f(to_check) { + self.swap(num_kept, next_to_check); + num_kept += 1; + } + next_to_check += 1; + } + self.truncate(num_kept); + } +} +// TODO: This is a temporary KVVec-specific implementation. It should be replaced with a generic +// `shrink_to()` for `impl Vec` that uses `A::realloc()` once the +// underlying allocators properly support shrinking via realloc. +impl Vec { + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// For `kmalloc` allocations, this delegates to `realloc()`, which decides whether + /// shrinking is worthwhile. For `vmalloc` allocations, shrinking only occurs if the + /// operation would free at least one page of memory, and performs a deep copy since + /// `vrealloc` does not yet support in-place shrinking. + /// + /// # Examples + /// + /// ``` + /// // Allocate enough capacity to span multiple pages. + /// let elements_per_page = kernel::page::PAGE_SIZE / core::mem::size_of::(); + /// let mut v = KVVec::with_capacity(elements_per_page * 4, GFP_KERNEL)?; + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// + /// v.shrink_to(0, GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn shrink_to(&mut self, min_capacity: usize, flags: Flags) -> Result<(), AllocError> { + let target_cap = core::cmp::max(self.len(), min_capacity); + + if self.capacity() <= target_cap { + return Ok(()); + } + + if Self::is_zst() { + return Ok(()); + } + + // For kmalloc allocations, delegate to realloc() and let the allocator decide + // whether shrinking is worthwhile. + // + // SAFETY: `self.ptr` points to a valid `KVmalloc` allocation. + if !unsafe { bindings::is_vmalloc_addr(self.ptr.as_ptr().cast()) } { + let new_layout = ArrayLayout::::new(target_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `self.ptr` is valid and was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + KVmalloc::realloc( + Some(self.ptr.cast()), + new_layout.into(), + self.layout.into(), + flags, + NumaNode::NO_NODE, + )? + }; + + self.ptr = ptr.cast(); + self.layout = new_layout; + return Ok(()); + } + + // Only shrink if we would free at least one page. + let current_size = self.capacity() * core::mem::size_of::(); + let target_size = target_cap * core::mem::size_of::(); + let current_pages = current_size.div_ceil(PAGE_SIZE); + let target_pages = target_size.div_ceil(PAGE_SIZE); + + if current_pages <= target_pages { + return Ok(()); + } + + if target_cap == 0 { + if !self.layout.is_empty() { + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + } + self.ptr = NonNull::dangling(); + self.layout = ArrayLayout::empty(); + return Ok(()); + } + + // SAFETY: `target_cap <= self.capacity()` and original capacity was valid. + let new_layout = unsafe { ArrayLayout::::new_unchecked(target_cap) }; + + let new_ptr = KVmalloc::alloc(new_layout.into(), flags, NumaNode::NO_NODE)?; + + // SAFETY: + // - `self.as_ptr()` is valid for reads of `self.len()` elements of `T`. + // - `new_ptr` is valid for writes of at least `target_cap >= self.len()` elements. + // - The two allocations do not overlap since `new_ptr` is freshly allocated. + // - Both pointers are properly aligned for `T`. + unsafe { + ptr::copy_nonoverlapping(self.as_ptr(), new_ptr.as_ptr().cast::(), self.len()) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + + self.ptr = new_ptr.cast::(); + self.layout = new_layout; + + Ok(()) + } +} + +impl Vec { + /// Extend the vector by `n` clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_with(3, 5, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5]); + /// + /// v.extend_with(2, 8, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// v.extend_with(0, 3, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { + if n == 0 { + return Ok(()); + } + + self.reserve(n, flags)?; + + let spare = self.spare_capacity_mut(); + + for item in spare.iter_mut().take(n - 1) { + item.write(value.clone()); + } + + // We can write the last element directly without cloning needlessly. + spare[n - 1].write(value); + + // SAFETY: + // - `self.len() + n <= self.capacity()` due to the call to reserve above, + // - the loop and the line above initialized the next `n` elements. + unsafe { self.inc_len(n) }; + + Ok(()) + } + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> { + self.reserve(other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: + // - `other.len()` spare entries have just been initialized, so it is safe to increase + // the length by the same number. + // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve` + // call. + unsafe { self.inc_len(other.len()) }; + Ok(()) + } + + /// Create a new `Vec` and extend it by `n` clones of `value`. + pub fn from_elem(value: T, n: usize, flags: Flags) -> Result { + let mut v = Self::with_capacity(n, flags)?; + + v.extend_with(n, value, flags)?; + + Ok(v) + } + + /// Resizes the [`Vec`] so that `len` is equal to `new_len`. + /// + /// If `new_len` is smaller than `len`, the `Vec` is [`Vec::truncate`]d. + /// If `new_len` is larger, each new slot is filled with clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.resize(1, 42, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.resize(3, 42, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 42, 42]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn resize(&mut self, new_len: usize, value: T, flags: Flags) -> Result<(), AllocError> { + match new_len.checked_sub(self.len()) { + Some(n) => self.extend_with(n, value, flags), + None => { + self.truncate(new_len); + Ok(()) + } + } + } +} + +impl Drop for Vec +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant. + unsafe { + ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr(), + self.len, + )) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.ptr.cast(), self.layout.into()) }; + } +} + +impl From> for Vec +where + A: Allocator, +{ + fn from(b: Box<[T; N], A>) -> Vec { + let len = b.len(); + let ptr = Box::into_raw(b); + + // SAFETY: + // - `b` has been allocated with `A`, + // - `ptr` fulfills the alignment requirements for `T`, + // - `ptr` points to memory with at least a size of `size_of::() * len`, + // - all elements within `b` are initialized values of `T`, + // - `len` does not exceed `isize::MAX`. + unsafe { Vec::from_raw_parts(ptr.cast(), len, len) } + } +} + +impl Default for Vec { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Deref for Vec +where + A: Allocator, +{ + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +impl DerefMut for Vec +where + A: Allocator, +{ + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// struct Foo>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow<[T]> for Vec +where + A: Allocator, +{ + fn borrow(&self) -> &[T] { + self.as_slice() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// struct Foo>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let mut arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&mut arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut<[T]> for Vec +where + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Eq for Vec where A: Allocator {} + +impl, A> Index for Vec +where + A: Allocator, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +impl, A> IndexMut for Vec +where + A: Allocator, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +macro_rules! impl_slice_eq { + ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => { + $( + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + } + )* + } +} + +impl_slice_eq! { + [A1: Allocator, A2: Allocator] Vec, Vec, + [A: Allocator] Vec, &[U], + [A: Allocator] Vec, &mut [U], + [A: Allocator] &[T], Vec, + [A: Allocator] &mut [T], Vec, + [A: Allocator] Vec, [U], + [A: Allocator] [T], Vec, + [A: Allocator, const N: usize] Vec, [U; N], + [A: Allocator, const N: usize] Vec, &[U; N], +} + +impl<'a, T, A> IntoIterator for &'a Vec +where + A: Allocator, +{ + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec +where + A: Allocator, +{ + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// # Examples +/// +/// ``` +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; +/// +/// let mut vec = VVec::::new(); +/// +/// assert!(vec.page_iter().next().is_none()); +/// +/// vec.reserve(PAGE_SIZE, GFP_KERNEL)?; +/// +/// let page = vec.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VVec { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.ptr.cast(); + let size = self.layout.size(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} + +/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. +/// +/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the +/// [`IntoIterator`] trait). +/// +/// # Examples +/// +/// ``` +/// let v = kernel::kvec![0, 1, 2]?; +/// let iter = v.into_iter(); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct IntoIter { + ptr: *mut T, + buf: NonNull, + len: usize, + layout: ArrayLayout, + _p: PhantomData, +} + +impl IntoIter +where + A: Allocator, +{ + fn into_raw_parts(self) -> (*mut T, NonNull, usize, usize) { + let me = ManuallyDrop::new(self); + let ptr = me.ptr; + let buf = me.buf; + let len = me.len; + let cap = me.layout.len(); + (ptr, buf, len, cap) + } + + /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`. + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// + /// let v = it.collect(GFP_KERNEL); + /// assert_eq!(v, [2, 3]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Implementation details + /// + /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait + /// in the kernel, namely: + /// + /// - Rust's specialization feature is unstable. This prevents us to optimize for the special + /// case where `I::IntoIter` equals `Vec`'s `IntoIter` type. + /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` + /// doesn't require this type to be `'static`. + /// - `FromIterator::from_iter` does return `Self` instead of `Result`, hence + /// we can't properly handle allocation failures. + /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation + /// flags. + /// + /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a + /// `Vec` again. + /// + /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing + /// buffer. However, this backing buffer may be shrunk to the actual count of elements. + pub fn collect(self, flags: Flags) -> Vec { + let old_layout = self.layout; + let (mut ptr, buf, len, mut cap) = self.into_raw_parts(); + let has_advanced = ptr != buf.as_ptr(); + + if has_advanced { + // Copy the contents we have advanced to at the beginning of the buffer. + // + // SAFETY: + // - `ptr` is valid for reads of `len * size_of::()` bytes, + // - `buf.as_ptr()` is valid for writes of `len * size_of::()` bytes, + // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to + // each other, + // - both `ptr` and `buf.ptr()` are properly aligned. + unsafe { ptr::copy(ptr, buf.as_ptr(), len) }; + ptr = buf.as_ptr(); + + // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()` by the type + // invariant. + let layout = unsafe { ArrayLayout::::new_unchecked(len) }; + + // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed by + // the type invariant to be smaller than `cap`. Depending on `realloc` this operation + // may shrink the buffer or leave it as it is. + ptr = match unsafe { + A::realloc( + Some(buf.cast()), + layout.into(), + old_layout.into(), + flags, + NumaNode::NO_NODE, + ) + } { + // If we fail to shrink, which likely can't even happen, continue with the existing + // buffer. + Err(_) => ptr, + Ok(ptr) => { + cap = len; + ptr.as_ptr().cast() + } + }; + } + + // SAFETY: If the iterator has been advanced, the advanced elements have been copied to + // the beginning of the buffer and `len` has been adjusted accordingly. + // + // - `ptr` is guaranteed to point to the start of the backing buffer. + // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`. + // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original + // `Vec`. + unsafe { Vec::from_raw_parts(ptr, len, cap) } + } +} + +impl Iterator for IntoIter +where + A: Allocator, +{ + type Item = T; + + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// assert_eq!(it.next(), Some(2)); + /// assert_eq!(it.next(), Some(3)); + /// assert_eq!(it.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn next(&mut self) -> Option { + if self.len == 0 { + return None; + } + + let current = self.ptr; + + // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr` + // by one guarantees that. + unsafe { self.ptr = self.ptr.add(1) }; + + self.len -= 1; + + // SAFETY: `current` is guaranteed to point at a valid element within the buffer. + Some(unsafe { current.read() }) + } + + /// # Examples + /// + /// ``` + /// let v: KVec = kernel::kvec![1, 2, 3]?; + /// let mut iter = v.into_iter(); + /// let size = iter.size_hint().0; + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 1); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 2); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 3); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl Drop for IntoIter +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) }; + + // SAFETY: + // - `self.buf` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.buf.cast(), self.layout.into()) }; + } +} + +impl IntoIterator for Vec +where + A: Allocator, +{ + type Item = T; + type IntoIter = IntoIter; + + /// Consumes the `Vec` and creates an `Iterator`, which moves each value out of the + /// vector (from start to end). + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2]?; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some(1)); + /// assert_eq!(v_iter.next(), Some(2)); + /// assert_eq!(v_iter.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// let v = kernel::kvec![]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, None); + /// + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + let buf = self.ptr; + let layout = self.layout; + let (ptr, len, _) = self.into_raw_parts(); + + IntoIter { + ptr, + buf, + len, + layout, + _p: PhantomData::, + } + } +} + +/// An iterator that owns all items in a vector, but does not own its allocation. +/// +/// # Invariants +/// +/// Every `&mut T` returned by the iterator references a `T` that the iterator may take ownership +/// of. +pub struct DrainAll<'vec, T> { + elements: slice::IterMut<'vec, T>, +} + +impl<'vec, T> Iterator for DrainAll<'vec, T> { + type Item = T; + + fn next(&mut self) -> Option { + let elem: *mut T = self.elements.next()?; + // SAFETY: By the type invariants, we may take ownership of this value. + Some(unsafe { elem.read() }) + } + + fn size_hint(&self) -> (usize, Option) { + self.elements.size_hint() + } +} + +impl<'vec, T> Drop for DrainAll<'vec, T> { + fn drop(&mut self) { + if core::mem::needs_drop::() { + let iter = core::mem::take(&mut self.elements); + let ptr: *mut [T] = iter.into_slice(); + // SAFETY: By the type invariants, we own these values so we may destroy them. + unsafe { ptr::drop_in_place(ptr) }; + } + } +} + +#[cfg(CONFIG_RUST_KVEC_KUNIT_TEST)] +#[macros::kunit_tests(rust_kvec)] +mod tests { + use super::*; + use crate::prelude::*; + + #[test] + fn test_kvec_retain() { + /// Verify correctness for one specific function. + #[expect(clippy::needless_range_loop)] + fn verify(c: &[bool]) { + let mut vec1: KVec = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap(); + let mut vec2: KVec = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap(); + + for i in 0..c.len() { + vec1.push_within_capacity(i).unwrap(); + if c[i] { + vec2.push_within_capacity(i).unwrap(); + } + } + + vec1.retain(|i| c[*i]); + + assert_eq!(vec1, vec2); + } + + /// Add one to a binary integer represented as a boolean array. + fn add(value: &mut [bool]) { + let mut carry = true; + for v in value { + let new_v = carry != *v; + carry = carry && *v; + *v = new_v; + } + } + + // This boolean array represents a function from index to boolean. We check that `retain` + // behaves correctly for all possible boolean arrays of every possible length less than + // ten. + let mut func = KVec::with_capacity(10, GFP_KERNEL).unwrap(); + for len in 0..10 { + for _ in 0u32..1u32 << len { + verify(&func); + add(&mut func); + } + func.push_within_capacity(false).unwrap(); + } + } + + #[test] + fn test_kvvec_shrink_to() { + use crate::page::PAGE_SIZE; + + // Create a vector with capacity spanning multiple pages. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add a few elements. + v.push(1, GFP_KERNEL).unwrap(); + v.push(2, GFP_KERNEL).unwrap(); + v.push(3, GFP_KERNEL).unwrap(); + + let initial_capacity = v.capacity(); + assert!(initial_capacity >= PAGE_SIZE * 4); + + // Shrink to a capacity that would free at least one page. + v.shrink_to(PAGE_SIZE, GFP_KERNEL).unwrap(); + + // Capacity should have been reduced. + assert!(v.capacity() < initial_capacity); + assert!(v.capacity() >= PAGE_SIZE); + + // Elements should be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + + // Shrink to zero (should shrink to len). + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Capacity should be at least the length. + assert!(v.capacity() >= v.len()); + + // Elements should still be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + } + + #[test] + fn test_kvvec_shrink_to_empty() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity but no elements. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + assert!(v.is_empty()); + + // Shrink empty vector to zero. + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Should have freed the allocation. + assert_eq!(v.capacity(), 0); + assert!(v.is_empty()); + } + + #[test] + fn test_kvvec_shrink_to_no_op() { + use crate::page::PAGE_SIZE; + + // Create a small vector. + let mut v = KVVec::::with_capacity(PAGE_SIZE, GFP_KERNEL).unwrap(); + v.push(1, GFP_KERNEL).unwrap(); + + let capacity_before = v.capacity(); + + // Try to shrink to a capacity larger than current - should be no-op. + v.shrink_to(capacity_before + 100, GFP_KERNEL).unwrap(); + + assert_eq!(v.capacity(), capacity_before); + assert_eq!(v.len(), 1); + assert_eq!(v[0], 1); + } + + #[test] + fn test_kvvec_shrink_to_respects_min_capacity() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add some elements. + for i in 0..10u8 { + v.push(i, GFP_KERNEL).unwrap(); + } + + // Shrink to a min_capacity larger than length. + let min_cap = PAGE_SIZE * 2; + v.shrink_to(min_cap, GFP_KERNEL).unwrap(); + + // Capacity should be at least min_capacity. + assert!(v.capacity() >= min_cap); + + // All elements preserved. + assert_eq!(v.len(), 10); + for i in 0..10u8 { + assert_eq!(v[i as usize], i); + } + } +} diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs new file mode 100644 index 000000000..aaca64465 --- /dev/null +++ b/rust/kernel/alloc/kvec/errors.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Errors for the [`Vec`] type. + +use crate::{ + fmt, + prelude::*, // +}; + +/// Error type for [`Vec::push_within_capacity`]. +pub struct PushError(pub T); + +impl fmt::Debug for PushError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Not enough capacity") + } +} + +impl From> for Error { + #[inline] + fn from(_: PushError) -> Error { + // Returning ENOMEM isn't appropriate because the system is not out of memory. The vector + // is just full and we are refusing to resize it. + EINVAL + } +} + +/// Error type for [`Vec::remove`]. +pub struct RemoveError; + +impl fmt::Debug for RemoveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Index out of bounds") + } +} + +impl From for Error { + #[inline] + fn from(_: RemoveError) -> Error { + EINVAL + } +} + +/// Error type for [`Vec::insert_within_capacity`]. +pub enum InsertError { + /// The value could not be inserted because the index is out of bounds. + IndexOutOfBounds(T), + /// The value could not be inserted because the vector is out of capacity. + OutOfCapacity(T), +} + +impl fmt::Debug for InsertError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + InsertError::IndexOutOfBounds(_) => write!(f, "Index out of bounds"), + InsertError::OutOfCapacity(_) => write!(f, "Not enough capacity"), + } + } +} + +impl From> for Error { + #[inline] + fn from(_: InsertError) -> Error { + EINVAL + } +} diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs new file mode 100644 index 000000000..62a459c66 --- /dev/null +++ b/rust/kernel/alloc/layout.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory layout. +//! +//! Custom layout types extending or improving [`Layout`]. + +use core::{ + alloc::Layout, + marker::PhantomData, // +}; + +/// Error when constructing an [`ArrayLayout`]. +pub struct LayoutError; + +/// A layout for an array `[T; n]`. +/// +/// # Invariants +/// +/// - `len * size_of::() <= isize::MAX`. +pub struct ArrayLayout { + len: usize, + _phantom: PhantomData T>, +} + +impl Clone for ArrayLayout { + fn clone(&self) -> Self { + *self + } +} +impl Copy for ArrayLayout {} + +const ISIZE_MAX: usize = isize::MAX as usize; + +impl ArrayLayout { + /// Creates a new layout for `[T; 0]`. + pub const fn empty() -> Self { + // INVARIANT: `0 * size_of::() <= isize::MAX`. + Self { + len: 0, + _phantom: PhantomData, + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Errors + /// + /// When `len * size_of::()` overflows or when `len * size_of::() > isize::MAX`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::alloc::layout::{ + /// # ArrayLayout, + /// # LayoutError, // + /// # }; + /// let layout = ArrayLayout::::new(15)?; + /// assert_eq!(layout.len(), 15); + /// + /// // Errors because `len * size_of::()` overflows. + /// let layout = ArrayLayout::::new(isize::MAX as usize); + /// assert!(layout.is_err()); + /// + /// // Errors because `len * size_of::() > isize::MAX`, + /// // even though `len < isize::MAX`. + /// let layout = ArrayLayout::::new(isize::MAX as usize / 2); + /// assert!(layout.is_err()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub const fn new(len: usize) -> Result { + match len.checked_mul(core::mem::size_of::()) { + Some(size) if size <= ISIZE_MAX => { + // INVARIANT: We checked above that `len * size_of::() <= isize::MAX`. + Ok(Self { + len, + _phantom: PhantomData, + }) + } + _ => Err(LayoutError), + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Safety + /// + /// `len` must be a value, for which `len * size_of::() <= isize::MAX` is true. + pub const unsafe fn new_unchecked(len: usize) -> Self { + // INVARIANT: By the safety requirements of this function + // `len * size_of::() <= isize::MAX`. + Self { + len, + _phantom: PhantomData, + } + } + + /// Returns the number of array elements represented by this layout. + pub const fn len(&self) -> usize { + self.len + } + + /// Returns `true` when no array elements are represented by this layout. + pub const fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the size of the [`ArrayLayout`] in bytes. + pub const fn size(&self) -> usize { + self.len() * core::mem::size_of::() + } +} + +impl From> for Layout { + fn from(value: ArrayLayout) -> Self { + let res = Layout::array::(value.len); + // SAFETY: By the type invariant of `ArrayLayout` we have + // `len * size_of::() <= isize::MAX` and thus the result must be `Ok`. + unsafe { res.unwrap_unchecked() } + } +} diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs new file mode 100644 index 000000000..60dfbec8f --- /dev/null +++ b/rust/kernel/auxiliary.rs @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the auxiliary bus. +//! +//! C header: [`include/linux/auxiliary_bus.h`](srctree/include/linux/auxiliary_bus.h) + +use crate::{ + bindings, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // + }, + ThisModule, // +}; +use core::{ + any::TypeId, + marker::PhantomData, + mem::offset_of, + pin::Pin, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +/// An adapter for the registration of auxiliary drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::auxiliary_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct auxiliary_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::auxiliary_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + adrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct auxiliary_driver` on initialization. + unsafe { + (*adrv.get()).name = name.as_char_ptr(); + (*adrv.get()).probe = Some(Self::probe_callback); + (*adrv.get()).remove = Some(Self::remove_callback); + (*adrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__auxiliary_driver_register(adrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(adrv: &Opaque) { + // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::auxiliary_driver_unregister(adrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback( + adev: *mut bindings::auxiliary_device, + id: *const bindings::auxiliary_device_id, + ) -> c_int { + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a + // `struct auxiliary_device`. + // + // INVARIANT: `adev` is valid for the duration of `probe_callback()`. + let adev = unsafe { &*adev.cast::>>() }; + + // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. + let info = unsafe { id.info_unchecked::() }; + + from_result(|| { + let data = T::probe(adev, info); + + adev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a + // `struct auxiliary_device`. + // + // INVARIANT: `adev` is valid for the duration of `remove_callback()`. + let adev = unsafe { &*adev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { adev.as_ref().drvdata_borrow::>() }; + + T::unbind(adev, data); + } +} + +/// Declares a kernel module that exposes a single auxiliary driver. +#[macro_export] +macro_rules! module_auxiliary_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::auxiliary::Adapter, { $($f)* }); + }; +} + +/// Abstraction for `bindings::auxiliary_device_id`. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::auxiliary_device_id); + +impl DeviceId { + /// Create a new [`DeviceId`] from name. + pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self { + let name = name.to_bytes_with_nul(); + let modname = modname.to_bytes_with_nul(); + + let mut id: bindings::auxiliary_device_id = pin_init::zeroed(); + let mut i = 0; + while i < modname.len() { + id.name[i] = modname[i]; + i += 1; + } + + // Reuse the space of the NULL terminator. + id.name[i - 1] = b'.'; + + let mut j = 0; + while j < name.len() { + id.name[i] = name[j]; + i += 1; + j += 1; + } + + Self(id) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `auxiliary_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::auxiliary_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = + core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); +} + +/// IdTable type for auxiliary drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a auxiliary `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! auxiliary_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("auxiliary", $crate::auxiliary::DeviceId, $($tt)*); + }; +} + +/// The auxiliary driver trait. +/// +/// Drivers must implement this trait in order to get an auxiliary driver registered. +pub trait Driver { + /// The type holding information about each device id supported by the driver. + /// + /// TODO: Use associated_type_defaults once stabilized: + /// + /// type IdInfo: 'static = (); + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// Auxiliary driver probe. + /// + /// Called when an auxiliary device is matches a corresponding driver. + fn probe<'bound>( + dev: &'bound Device>, + id_info: &'bound Self::IdInfo, + ) -> impl PinInit, Error> + 'bound; + + /// Auxiliary driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The auxiliary device representation. +/// +/// This structure represents the Rust abstraction for a C `struct auxiliary_device`. The +/// implementation abstracts the usage of an already existing C `struct auxiliary_device` within +/// Rust code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct auxiliary_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::auxiliary_device { + self.0.get() + } + + /// Returns the auxiliary device' id. + pub fn id(&self) -> u32 { + // SAFETY: By the type invariant `self.as_raw()` is a valid pointer to a + // `struct auxiliary_device`. + unsafe { (*self.as_raw()).id } + } +} + +impl Device { + /// Returns a bound reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device { + let parent = (**self).parent(); + + // SAFETY: A bound auxiliary device always has a bound parent device. + unsafe { parent.as_bound() } + } + + /// Returns the stored registration data as a pinned reference. + /// + /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`]. + /// + /// # Safety + /// + /// Callers must ensure that the lifetime shortening from the original `'static` storage to + /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee. + unsafe fn registration_data_pinned(&self) -> Result>> { + // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. + let ptr = unsafe { (*self.as_raw()).registration_data_rust }; + if ptr.is_null() { + dev_warn!( + self.as_ref(), + "No registration data set; parent is not a Rust driver.\n" + ); + return Err(ENOENT); + } + + // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`; + // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId` + // at the start of the allocation is valid regardless of `F`. + let type_id = unsafe { ptr.cast::().read() }; + if type_id != TypeId::of::() { + return Err(EINVAL); + } + + // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s + // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is + // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until + // `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::>>>::borrow(ptr) }; + + // SAFETY: `data` is a structurally pinned field of `RegistrationData`. + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } + + /// Access the registration data set by the registering (parent) driver through a closure. + /// + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned + /// reference to the registration data. + /// + /// For covariant types that implement [`trait@CovariantForLt`], prefer + /// [`registration_data`](Self::registration_data) which returns a direct reference. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data_with( + &self, + f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + ) -> Result { + // SAFETY: The HRTB closure prevents the caller from smuggling in references with a + // concrete short lifetime, making the round-trip from `'static` sound regardless of + // variance. + let pinned = unsafe { self.registration_data_pinned::()? }; + + Ok(f(pinned)) + } + + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees + /// that the lifetime shortening is sound. + /// + /// For non-covariant types, use the closure-based [`Self::registration_data_with`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data(&self) -> Result>> { + // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening + // from `'static` to `'_` performed by `registration_data_pinned` sound. + unsafe { self.registration_data_pinned::() } + } +} + +impl Device { + /// Returns a reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device { + // SAFETY: A `struct auxiliary_device` always has a parent. + unsafe { self.as_ref().parent().unwrap_unchecked() } + } + + extern "C" fn release(dev: *mut bindings::device) { + // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device` + // embedded in `struct auxiliary_device`. + let adev = unsafe { container_of!(dev, bindings::auxiliary_device, dev) }; + + // SAFETY: `adev` points to the memory that has been allocated in `Registration::new`, via + // `KBox::new(Opaque::::zeroed(), GFP_KERNEL)`. + let _ = unsafe { KBox::>::from_raw(adev.cast()) }; + } +} + +// SAFETY: `auxiliary::Device` is a transparent wrapper of `struct auxiliary_device`. +// The offset is guaranteed to point to a valid device field inside `auxiliary::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::auxiliary_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // CAST: `Self` a transparent wrapper of `bindings::auxiliary_device`. + let adev: *mut bindings::auxiliary_device = obj.cast().as_ptr(); + + // SAFETY: By the type invariant of `Self`, `adev` is a pointer to a valid + // `struct auxiliary_device`. + let dev = unsafe { addr_of_mut!((*adev).dev) }; + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(dev) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct auxiliary_device`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct auxiliary_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking. +#[repr(C)] +#[pin_data] +struct RegistrationData { + type_id: TypeId, + #[pin] + data: T, +} + +/// The registration of an auxiliary device. +/// +/// This type represents the registration of a [`struct auxiliary_device`]. When its parent device +/// is unbound, the corresponding auxiliary device will be unregistered from the system. +/// +/// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration +/// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). +/// +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and +/// [`Device::registration_data_with()`]. +/// +/// # Invariants +/// +/// `self.adev` always holds a valid pointer to an initialized and registered +/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a +/// valid `Pin>>>`. +pub struct Registration<'a, F: ForLt + 'static> { + adev: NonNull, + _phantom: PhantomData>, +} + +impl<'a, F: ForLt> Registration<'a, F> +where + for<'b> F::Of<'b>: Send + Sync, +{ + /// Create and register a new auxiliary device with the given registration data. + /// + /// The `data` is owned by the registration and can be accessed through the auxiliary device + /// via [`Device::registration_data()`]. + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + /// + /// If the registration data is `'static`, use the safe [`Registration::new()`] instead. + pub unsafe fn new_with_lt( + parent: &'a device::Device, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit, E>, + ) -> Result + where + Error: From, + { + let data = KBox::pin_init::( + try_pin_init!(RegistrationData { + type_id: TypeId::of::(), + data <- data, + }), + GFP_KERNEL, + )?; + + // SAFETY: `'a` is invariant (via `Registration`'s `PhantomData`). Lifetimes do not + // affect layout, so RegistrationData> and RegistrationData> + // have identical representation. + let data: Pin>>> = + unsafe { core::mem::transmute(data) }; + + let boxed: KBox> = KBox::zeroed(GFP_KERNEL)?; + let adev = boxed.get(); + + // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. + unsafe { + (*adev).dev.parent = parent.as_raw(); + (*adev).dev.release = Some(Device::release); + (*adev).name = name.as_char_ptr(); + (*adev).id = id; + (*adev).registration_data_rust = data.into_foreign(); + } + + // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, + // which has not been initialized yet. + unsafe { bindings::auxiliary_device_init(adev) }; + + // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be + // freed by `Device::release` when the last reference to the `struct auxiliary_device` + // is dropped. + let _ = KBox::into_raw(boxed); + + // SAFETY: + // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which + // has been initialized, + // - `modname.as_char_ptr()` is a NULL terminated string. + let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; + if ret != 0 { + // SAFETY: `registration_data` was set above via `into_foreign()`. + drop(unsafe { + Pin::>>>::from_foreign( + (*adev).registration_data_rust, + ) + }); + + // SAFETY: `adev` is guaranteed to be a valid pointer to a + // `struct auxiliary_device`, which has been initialized. + unsafe { bindings::auxiliary_device_uninit(adev) }; + + return Err(Error::from_errno(ret)); + } + + // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is + // called, which happens in `Self::drop()`. + Ok(Self { + // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated + // successfully. + adev: unsafe { NonNull::new_unchecked(adev) }, + _phantom: PhantomData, + }) + } + + /// Create and register a new auxiliary device with `'static` registration data. + /// + /// Safe variant of [`Registration::new_with_lt()`] for registration data that does not contain + /// borrowed references. + pub fn new( + parent: &'a device::Device, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit, E>, + ) -> Result + where + F::Of<'a>: 'static, + Error: From, + { + // SAFETY: `F::Of<'a>: 'static` guarantees the data contains no borrowed references, + // so forgetting the `Registration` cannot cause use-after-free. + unsafe { Self::new_with_lt(parent, name, id, modname, data) } + } +} + +impl Drop for Registration<'_, F> { + fn drop(&mut self) { + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered + // `struct auxiliary_device`. + unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) }; + + // SAFETY: `registration_data` was set in `new()` via `into_foreign()`. + drop(unsafe { + Pin::>>>::from_foreign( + (*self.adev.as_ptr()).registration_data_rust, + ) + }); + + // This drops the reference we acquired through `auxiliary_device_init()`. + // + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered + // `struct auxiliary_device`. + unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) }; + } +} + +// SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread. +unsafe impl Send for Registration<'_, F> where for<'a> F::Of<'a>: Send {} + +// SAFETY: `Registration` does not expose any methods or fields that need synchronization. +unsafe impl Sync for Registration<'_, F> where for<'a> F::Of<'a>: Send {} diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs new file mode 100644 index 000000000..a0d089423 --- /dev/null +++ b/rust/kernel/bitfield.rs @@ -0,0 +1,865 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for defining bitfields as Rust structures. +//! +//! The [`bitfield!`](kernel::bitfield!) macro declares integer types that are split into distinct +//! bit fields of arbitrary length. Each field is typed using [`Bounded`](kernel::num::Bounded) to +//! ensure values are properly validated and to avoid implicit data loss. +//! +//! # Example +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! bitfield! { +//! pub struct Rgb(u16) { +//! 15:11 blue; +//! 10:5 green; +//! 4:0 red; +//! } +//! } +//! +//! // Valid value for the `blue` field. +//! let blue = Bounded::::new::<0x18>(); +//! +//! // Setters can be chained. Values ranges are checked at compile-time. +//! let color = Rgb::zeroed() +//! // Compile-time bounds check of constant value. +//! .with_const_red::<0x10>() +//! .with_const_green::<0x1f>() +//! // A `Bounded` can also be passed. +//! .with_blue(blue); +//! +//! assert_eq!(color.red(), 0x10); +//! assert_eq!(color.green(), 0x1f); +//! assert_eq!(color.blue(), 0x18); +//! assert_eq!( +//! color.into_raw(), +//! (0x18 << Rgb::BLUE_SHIFT) + (0x1f << Rgb::GREEN_SHIFT) + 0x10, +//! ); +//! +//! // Convert to/from the backing storage type. +//! let raw: u16 = color.into(); +//! assert_eq!(Rgb::from(raw), color); +//! ``` +//! +//! # Syntax +//! +//! ```text +//! bitfield! { +//! #[attributes] +//! // Documentation for `Name`. +//! pub struct Name(storage_type) { +//! // `field_1` documentation. +//! hi:lo field_1; +//! // `field_2` documentation. +//! hi:lo field_2 => ConvertedType; +//! // `field_3` documentation. +//! hi:lo field_3 ?=> ConvertedType; +//! ... +//! } +//! } +//! ``` +//! +//! - `storage_type`: The underlying unsigned integer type ([`u8`], [`u16`], [`u32`], [`u64`]). +//! Signed integer storage types are not supported. +//! - `hi:lo`: Bit range (inclusive), where `hi >= lo`. +//! - `=> Type`: Optional infallible conversion (see [below](#infallible-conversion-)). +//! - `?=> Type`: Optional fallible conversion (see [below](#fallible-conversion-)). +//! - Documentation strings and attributes are optional. +//! +//! # Generated code +//! +//! Each field is internally represented as a [`Bounded`] parameterized by its bit width. Field +//! values can either be set/retrieved directly, or converted from/to another type. +//! +//! The use of [`Bounded`] for each field enforces bounds-checking (at build time or runtime) of +//! every value assigned to a field. This ensures that data is never accidentally truncated. +//! +//! The macro generates the bitfield type, [`From`] and [`Into`] implementations for its storage +//! type, as well as [`Debug`] and [`Zeroable`](pin_init::Zeroable) implementations. +//! +//! For each field, it also generates: +//! +//! - `field()`: Getter method for the field value. +//! - `with_field(value)`: Infallible setter; the argument type must fit within the field's width. +//! - `with_const_field::()`: `const` setter; the value is validated at compile time. +//! Usually shorter to use than `with_field` for constant values as it doesn't require +//! constructing a [`Bounded`]. +//! - `try_with_field(value)`: Fallible setter. Returns an error if the value is out of range. +//! - `FIELD_MASK`, `FIELD_SHIFT`, `FIELD_RANGE`: Constants for manual bit manipulation. +//! +//! # Reserved names for field identifiers +//! +//! Field identifiers are used to generate methods and associated constants on the bitfield type. +//! For a field named `field`, the macro may generate methods named `field`, `with_field`, +//! `with_const_field`, `try_with_field`, `__field` and `__with_field`, as well as constants named +//! `FIELD_MASK`, `FIELD_SHIFT` and `FIELD_RANGE`. +//! +//! Therefore, field identifiers must not use names that would collide with generated items for +//! any field in the same bitfield. The following prefixes are thus reserved for field identifiers: +//! +//! - `with_` +//! - `const_` +//! - `try_with_` +//! - `__` +//! +//! The field identifiers `from_raw`, `into_raw`, and `into` are also reserved. +//! +//! In addition, field identifiers should follow Rust `snake_case` conventions, since the associated +//! constants are generated by uppercasing the field name. +//! +//! # Implicit conversions +//! +//! Types that fit entirely within a field's bit width can be used directly with setters. For +//! example, [`bool`] works with single-bit fields, and [`u8`] works with 8-bit fields: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Flags(u32) { +//! 15:8 byte_field; +//! 0:0 flag; +//! } +//! } +//! +//! let flags = Flags::zeroed() +//! .with_byte_field(0x42_u8) +//! .with_flag(true); +//! +//! assert_eq!(flags.into_raw(), (0x42 << Flags::BYTE_FIELD_SHIFT) | 1); +//! ``` +//! +//! # Runtime bounds checking +//! +//! When a value is not known at compile time, use `try_with_field()` to check bounds at runtime: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Config(u8) { +//! 3:0 nibble; +//! } +//! } +//! +//! fn set_nibble(config: Config, value: u8) -> Result { +//! // Returns `EOVERFLOW` if `value > 0xf`. +//! config.try_with_nibble(value) +//! } +//! # Ok::<(), Error>(()) +//! ``` +//! +//! # Type conversion +//! +//! Fields can be automatically converted to/from a custom type using `=>` (infallible) or `?=>` +//! (fallible). The custom type must implement the appropriate [`From`] or [`TryFrom`] traits with +//! [`Bounded`]. +//! +//! ## Infallible conversion (`=>`) +//! +//! Use this when all possible bit patterns of a field map to valid values: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Power { +//! Off, +//! On, +//! } +//! +//! impl From> for Power { +//! fn from(v: Bounded) -> Self { +//! match *v { +//! 0 => Power::Off, +//! _ => Power::On, +//! } +//! } +//! } +//! +//! impl From for Bounded { +//! fn from(p: Power) -> Self { +//! (p as u32 != 0).into() +//! } +//! } +//! +//! bitfield! { +//! pub struct Control(u32) { +//! 0:0 power => Power; +//! } +//! } +//! +//! let ctrl = Control::zeroed().with_power(Power::On); +//! assert_eq!(ctrl.power(), Power::On); +//! ``` +//! +//! ## Fallible conversion (`?=>`) +//! +//! Use this when some bit patterns of a field are invalid. The getter returns a [`Result`]: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Mode { +//! Low = 0, +//! High = 1, +//! Auto = 2, +//! // 3 is invalid +//! } +//! +//! impl TryFrom> for Mode { +//! type Error = u32; +//! +//! fn try_from(v: Bounded) -> Result { +//! match *v { +//! 0 => Ok(Mode::Low), +//! 1 => Ok(Mode::High), +//! 2 => Ok(Mode::Auto), +//! n => Err(n), +//! } +//! } +//! } +//! +//! impl From for Bounded { +//! fn from(m: Mode) -> Self { +//! match m { +//! Mode::Low => Bounded::::new::<0>(), +//! Mode::High => Bounded::::new::<1>(), +//! Mode::Auto => Bounded::::new::<2>(), +//! } +//! } +//! } +//! +//! bitfield! { +//! pub struct Config(u32) { +//! 1:0 mode ?=> Mode; +//! } +//! } +//! +//! let cfg = Config::zeroed().with_mode(Mode::Auto); +//! assert_eq!(cfg.mode(), Ok(Mode::Auto)); +//! +//! // Invalid bit pattern returns an error. +//! assert_eq!(Config::from(0b11).mode(), Err(3)); +//! ``` +//! +//! # Bits outside of declared fields +//! +//! Bits of the storage type that are not part of any declared field are preserved by the setter +//! methods, and can only be modified through `from_raw` or the [`From`] implementation from the +//! storage type. +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Sparse(u8) { +//! 7:6 high; +//! // Bits 5:1 are not covered by any field. +//! 0:0 low; +//! } +//! } +//! +//! // Set the gap bits via `from_raw`, then mutate the declared fields. +//! let val = Sparse::from_raw(0b0010_1010) +//! .with_const_high::<0b11>() +//! .with_low(true); +//! +//! // Bits 5:1 are unchanged. +//! assert_eq!(val.into_raw(), 0b1110_1011); +//! ``` +//! +//! # Signed field values +//! +//! Bitfield storage types are unsigned. Since field getter methods return a [`Bounded`] of the +//! storage type, fields are also unsigned by default. +//! +//! If a field needs to encode a signed value, use a custom conversion type with `=>` or `?=>` to +//! perform the sign interpretation explicitly. +//! +//! [`Bounded`]: kernel::num::Bounded + +/// Defines a bitfield struct with bounds-checked accessors for individual bit ranges. +/// +/// See the [`mod@kernel::bitfield`] module for full documentation and examples. +#[macro_export] +macro_rules! bitfield { + // Entry point defining the bitfield struct, its implementations and its field accessors. + ( + $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!(@core + #[allow(non_camel_case_types)] + $(#[$attr])* $vis $name $storage + ); + $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); + }; + + // All rules below are helpers. + + // Defines the wrapper `$name` type and its conversions from/to the storage type. + (@core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { + $(#[$attr])* + #[repr(transparent)] + #[derive(Clone, Copy, PartialEq, Eq)] + $vis struct $name { + inner: $storage, + } + + #[allow(dead_code)] + impl $name { + /// Creates a bitfield from a raw value. + #[inline(always)] + $vis const fn from_raw(value: $storage) -> Self { + Self{ inner: value } + } + + /// Turns this bitfield into its raw value. + /// + /// This is similar to the [`From`] implementation, but is shorter to invoke in + /// most cases. + #[inline(always)] + $vis const fn into_raw(self) -> $storage { + self.inner + } + } + + // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. + unsafe impl ::pin_init::Zeroable for $name {} + + impl ::core::convert::From<$name> for $storage { + #[inline(always)] + fn from(val: $name) -> $storage { + val.into_raw() + } + } + + impl ::core::convert::From<$storage> for $name { + #[inline(always)] + fn from(val: $storage) -> $name { + Self::from_raw(val) + } + } + }; + + // Definitions requiring knowledge of individual fields: private and public field accessors, + // and `Debug` implementation. + (@fields $vis:vis $name:ident $storage:ty { + $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident + $(?=> $try_into_type:ty)? + $(=> $into_type:ty)? + ; + )* + } + ) => { + #[allow(dead_code)] + impl $name { + $( + $crate::bitfield!(@private_field_accessors $vis $name $storage : $hi:$lo $field); + $crate::bitfield!( + @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field + $(?=> $try_into_type)? + $(=> $into_type)? + ); + )* + } + + $crate::bitfield!(@debug $name { $($field;)* }); + }; + + // Private field accessors working with the exact `Bounded` type for the field. + ( + @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive = $lo..=$hi; + $vis const [<$field:upper _MASK>]: $storage = + ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); + $vis const [<$field:upper _SHIFT>]: u32 = $lo; + ); + + ::kernel::macros::paste!( + #[inline(always)] + fn [<__ $field>](self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { + // Left shift to align the field's MSB with the storage MSB. + const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); + // Right shift to move the top-aligned field to bit 0 of the storage. + const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; + + // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized + // output type. + let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( + self.inner << ALIGN_TOP + ); + val.shr::() + } + + #[inline(always)] + const fn [<__with_ $field>]( + mut self, + value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, + ) -> Self + { + const MASK: $storage = <$name>::[<$field:upper _MASK>]; + const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; + + let value = value.get() << SHIFT; + self.inner = (self.inner & !MASK) | value; + + self + } + ); + }; + + // Public accessors for fields infallibly (`=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:literal:$lo:literal $field:ident => $into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> $into_type + { + self.[<__ $field>]().into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields fallibly (`?=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::core::result::Result< + $try_into_type, + <$try_into_type as ::core::convert::TryFrom< + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + >>::Error + > + { + self.[<__ $field>]().try_into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $try_into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields not converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + { + self.[<__ $field>]() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the compile-time constant `VALUE`."] + #[inline(always)] + $vis const fn [](self) -> Self { + self.[<__with_ $field>]( + ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::() + ) + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> Self + where T: ::core::convert::Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, + { + self.[<__with_ $field>](value.into()) + } + + $(#[doc = $doc])* + #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> ::kernel::error::Result + where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, + { + Ok( + self.[<__with_ $field>]( + value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? + ) + ) + } + + ); + }; + + // `Debug` implementation. + (@debug $name:ident { $($field:ident;)* }) => { + impl ::kernel::fmt::Debug for $name { + #[inline] + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { + f.debug_struct(stringify!($name)) + .field("", &::kernel::prelude::fmt!("{:#x}", self.inner)) + $( + .field(stringify!($field), &self.$field()) + )* + .finish() + } + } + }; +} + +#[cfg(CONFIG_RUST_BITFIELD_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(rust_kernel_bitfield)] +mod tests { + use core::convert::TryFrom; + + use pin_init::Zeroable; + + use kernel::num::Bounded; + + // Enum types for testing `=>` and `?=>` conversions. + + #[derive(Debug, Clone, Copy, PartialEq)] + enum MemoryType { + Unmapped = 0, + Normal = 1, + Device = 2, + Reserved = 3, + } + + impl TryFrom> for MemoryType { + type Error = u64; + fn try_from(value: Bounded) -> Result { + match value.get() { + 0 => Ok(MemoryType::Unmapped), + 1 => Ok(MemoryType::Normal), + 2 => Ok(MemoryType::Device), + 3 => Ok(MemoryType::Reserved), + _ => Err(value.get()), + } + } + } + + impl From for Bounded { + #[inline(always)] + fn from(mt: MemoryType) -> Bounded { + Bounded::from_expr(mt as u64) + } + } + + #[derive(Debug, Clone, Copy, PartialEq)] + enum Priority { + Low = 0, + Medium = 1, + High = 2, + Critical = 3, + } + + impl From> for Priority { + fn from(value: Bounded) -> Self { + match value & 0x3 { + 0 => Priority::Low, + 1 => Priority::Medium, + 2 => Priority::High, + _ => Priority::Critical, + } + } + } + + impl From for Bounded { + #[inline(always)] + fn from(p: Priority) -> Bounded { + Bounded::from_expr(p as u16) + } + } + + bitfield! { + struct TestU64(u64) { + 63:63 field_63; + 61:52 field_61_52; + 51:16 field_51_16; + 15:12 field_15_12 ?=> MemoryType; + 11:9 field_11_9; + 1:1 field_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU16(u16) { + 15:8 field_15_8; + 7:4 field_7_4; // Partial overlap with `field_5_4`. + 5:4 field_5_4 => Priority; + 3:1 field_3_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU8(u8) { + 7:0 field_7_0; // Full byte overlap. + 7:4 field_7_4; + 3:2 field_3_2; + 1:1 field_1; + 0:0 field_0; + } + } + + // Single and multi-bit fields basic access. + #[test] + fn test_basic_access() { + // `TestU64`. + let mut val = TestU64::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + val = val.with_field_1(false); + assert!(!val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_11_9::<0x5>(); + assert_eq!(val.field_11_9(), 0x5); + assert_eq!(val.into_raw(), 0xA01); + + val = val.with_const_field_51_16::<0x123456>(); + assert_eq!(val.field_51_16(), 0x123456); + assert_eq!(val.into_raw(), 0x0012_3456_0A01); + + const MAX_FIELD_51_16: u64 = ::kernel::bits::genmask_u64(0..=35); + val = val.with_const_field_51_16::<{ MAX_FIELD_51_16 }>(); + assert_eq!(val.field_51_16(), MAX_FIELD_51_16); + + val = val.with_const_field_61_52::<0x3FF>(); + assert_eq!(val.field_61_52(), 0x3FF); + + val = val.with_field_63(true); + assert!(val.field_63().into_bool()); + + // `TestU16`. + let mut val = TestU16::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_3_1::<0x5>(); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.into_raw(), 0xB); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAB); + + val = val.with_const_field_15_8::<0x42>(); + assert_eq!(val.field_15_8(), 0x42); + assert_eq!(val.into_raw(), 0x42AB); + + // `TestU8`. + let mut val = TestU8::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x3); + + val = val.with_const_field_3_2::<0x3>(); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.into_raw(), 0xF); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAF); + } + + // `=>` infallible conversion. + #[test] + fn test_infallible_conversion() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::Low); + assert_eq!(val.field_5_4(), Priority::Low); + assert_eq!(val.into_raw() & 0x30, 0x00); + + val = val.with_field_5_4(Priority::Medium); + assert_eq!(val.field_5_4(), Priority::Medium); + assert_eq!(val.into_raw() & 0x30, 0x10); + + val = val.with_field_5_4(Priority::High); + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.into_raw() & 0x30, 0x20); + + val = val.with_field_5_4(Priority::Critical); + assert_eq!(val.field_5_4(), Priority::Critical); + assert_eq!(val.into_raw() & 0x30, 0x30); + } + + // `?=>` fallible conversion. + #[test] + fn test_fallible_conversion() { + let mut val = TestU64::zeroed(); + + val = val.with_field_15_12(MemoryType::Unmapped); + assert_eq!(val.field_15_12(), Ok(MemoryType::Unmapped)); + val = val.with_field_15_12(MemoryType::Normal); + assert_eq!(val.field_15_12(), Ok(MemoryType::Normal)); + val = val.with_field_15_12(MemoryType::Device); + assert_eq!(val.field_15_12(), Ok(MemoryType::Device)); + val = val.with_field_15_12(MemoryType::Reserved); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + + // `field_15_12` is 4 bits wide (0-15); `MemoryType` only covers 0-3, so 4-15 return `Err`. + let raw = (val.into_raw() & !::kernel::bits::genmask_u64(12..=15)) | (0x7 << 12); + assert_eq!(TestU64::from_raw(raw).field_15_12(), Err(0x7)); + } + + // Test that setting an overlapping field affects the overlapped one as expected. + #[test] + fn test_overlapping_fields() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::High); // High == 2 == 0b10. + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.field_7_4(), 0x2); // Bits 7:6 == 0, bits 5:4 == 0b10. + + val = val.with_const_field_7_4::<0xF>(); + assert_eq!(val.field_7_4(), 0xF); + assert_eq!(val.field_5_4(), Priority::Critical); // Bits 5:4 == 0b11. + + // `field_7_0` should encompass all other fields. + let mut val = TestU8::zeroed() + .with_field_0(true) + .with_field_1(true) + .with_const_field_3_2::<0x3>() + .with_const_field_7_4::<0xA>(); + assert_eq!(val.into_raw(), 0xAF); + + val = val.with_field_7_0(0x55); + assert_eq!(val.field_7_0(), 0x55); + assert!(val.field_0().into_bool()); + assert!(!val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x1); + assert_eq!(val.field_7_4(), 0x5); + } + + // Checks that bits not mapped to any field are left untouched. + #[test] + fn test_unallocated_bits() { + let gap_bits = (1u64 << 62) | 0x1FC; + + let set_all_fields = |val: TestU64| { + val.with_field_63(true) + .with_const_field_61_52::<0x155>() + .with_const_field_51_16::<0x123456>() + .with_field_15_12(MemoryType::Device) + .with_const_field_11_9::<0x5>() + .with_field_1(true) + .with_field_0(true) + }; + + // Gap bits to 0. + let val = set_all_fields(TestU64::from_raw(0)); + assert_eq!(val.into_raw() & gap_bits, 0); + + // Gap bits to 1. + let val = set_all_fields(TestU64::from_raw(gap_bits)); + assert_eq!(val.into_raw() & gap_bits, gap_bits); + } + + #[test] + fn test_try_with() { + let val = TestU64::zeroed().try_with_field_51_16(0x123456).unwrap(); + assert_eq!(val.field_51_16(), 0x123456); + + let err = TestU64::zeroed().try_with_field_51_16(u64::MAX); + assert_eq!(err, Err(::kernel::error::code::EOVERFLOW)); + + let val = TestU64::zeroed() + .try_with_field_51_16(0xABCDEF) + .and_then(|p| p.try_with_field_0(1)) + .unwrap(); + assert_eq!(val.field_51_16(), 0xABCDEF); + assert!(val.field_0().into_bool()); + } + + // `from_raw`/`into_raw` and `From`/`Into` round-trips. + #[test] + fn test_raw() { + let raw: u64 = 0xBFF0_0000_3123_3E03; + let val = TestU64::from_raw(raw); + assert_eq!(u64::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_11_9(), 0x7); + assert_eq!(val.field_51_16(), 0x3123); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + assert_eq!(val.field_61_52(), 0x3FF); + assert!(val.field_63().into_bool()); + + let raw: u16 = 0x42AB; + let val = TestU16::from_raw(raw); + assert_eq!(u16::from(val), raw); + assert!(val.field_0().into_bool()); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_15_8(), 0x42); + + let raw: u8 = 0xAF; + let val = TestU8::from_raw(raw); + assert_eq!(u8::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_7_0(), 0xAF); + } +} diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs new file mode 100644 index 000000000..b27e0ec80 --- /dev/null +++ b/rust/kernel/bitmap.rs @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! Rust API for bitmap. +//! +//! C headers: [`include/linux/bitmap.h`](srctree/include/linux/bitmap.h). + +use crate::alloc::{AllocError, Flags}; +use crate::bindings; +#[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] +use crate::pr_err; +use core::ptr::NonNull; + +/// Represents a C bitmap. Wraps underlying C bitmap API. +/// +/// # Invariants +/// +/// Must reference a `[c_ulong]` long enough to fit `data.len()` bits. +#[cfg_attr(CONFIG_64BIT, repr(align(8)))] +#[cfg_attr(not(CONFIG_64BIT), repr(align(4)))] +pub struct Bitmap { + data: [()], +} + +impl Bitmap { + /// Borrows a C bitmap. + /// + /// # Safety + /// + /// * `ptr` holds a non-null address of an initialized array of `unsigned long` + /// that is large enough to hold `nbits` bits. + /// * the array must not be freed for the lifetime of this [`Bitmap`] + /// * concurrent access only happens through atomic operations + pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap { + let data: *const [()] = core::ptr::slice_from_raw_parts(ptr.cast(), nbits); + // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // SAFETY: + // The caller guarantees that `data` (derived from `ptr` and `nbits`) + // points to a valid, initialized, and appropriately sized memory region + // that will not be freed for the lifetime 'a. + // We are casting `*const [()]` to `*const Bitmap`. The `Bitmap` + // struct is a ZST with a `data: [()]` field. This means its layout + // is compatible with a slice of `()`, and effectively it's a "thin pointer" + // (its size is 0 and alignment is 1). The `slice_from_raw_parts` + // function correctly encodes the length (number of bits, not elements) + // into the metadata of the fat pointer. Therefore, dereferencing this + // pointer as `&Bitmap` is safe given the caller's guarantees. + unsafe { &*(data as *const Bitmap) } + } + + /// Borrows a C bitmap exclusively. + /// + /// # Safety + /// + /// * `ptr` holds a non-null address of an initialized array of `unsigned long` + /// that is large enough to hold `nbits` bits. + /// * the array must not be freed for the lifetime of this [`Bitmap`] + /// * no concurrent access may happen. + pub unsafe fn from_raw_mut<'a>(ptr: *mut usize, nbits: usize) -> &'a mut Bitmap { + let data: *mut [()] = core::ptr::slice_from_raw_parts_mut(ptr.cast(), nbits); + // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // SAFETY: + // The caller guarantees that `data` (derived from `ptr` and `nbits`) + // points to a valid, initialized, and appropriately sized memory region + // that will not be freed for the lifetime 'a. + // Furthermore, the caller guarantees no concurrent access will happen, + // which upholds the exclusivity requirement for a mutable reference. + // Similar to `from_raw`, casting `*mut [()]` to `*mut Bitmap` is + // safe because `Bitmap` is a ZST with a `data: [()]` field, + // making its layout compatible with a slice of `()`. + unsafe { &mut *(data as *mut Bitmap) } + } + + /// Returns a raw pointer to the backing [`Bitmap`]. + pub fn as_ptr(&self) -> *const usize { + core::ptr::from_ref::(self).cast::() + } + + /// Returns a mutable raw pointer to the backing [`Bitmap`]. + pub fn as_mut_ptr(&mut self) -> *mut usize { + core::ptr::from_mut::(self).cast::() + } + + /// Returns length of this [`Bitmap`]. + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.data.len() + } +} + +/// Holds either a pointer to array of `unsigned long` or a small bitmap. +#[repr(C)] +union BitmapRepr { + bitmap: usize, + ptr: NonNull, +} + +macro_rules! bitmap_assert { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + } +} + +macro_rules! bitmap_assert_return { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + if !($cond) { + pr_err!($($arg)*); + return + } + } +} + +/// Represents an owned bitmap. +/// +/// Wraps underlying C bitmap API. See [`Bitmap`] for available +/// methods. +/// +/// # Examples +/// +/// Basic usage +/// +/// ``` +/// use kernel::alloc::flags::GFP_KERNEL; +/// use kernel::bitmap::BitmapVec; +/// +/// let mut b = BitmapVec::new(16, GFP_KERNEL)?; +/// +/// assert_eq!(16, b.len()); +/// for i in 0..16 { +/// if i % 4 == 0 { +/// b.set_bit(i); +/// } +/// } +/// assert_eq!(Some(0), b.next_bit(0)); +/// assert_eq!(Some(1), b.next_zero_bit(0)); +/// assert_eq!(Some(4), b.next_bit(1)); +/// assert_eq!(Some(5), b.next_zero_bit(4)); +/// assert_eq!(Some(12), b.last_bit()); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * `nbits` is `<= MAX_LEN`. +/// * if `nbits <= MAX_INLINE_LEN`, then `repr` is a `usize`. +/// * otherwise, `repr` holds a non-null pointer to an initialized +/// array of `unsigned long` that is large enough to hold `nbits` bits. +pub struct BitmapVec { + /// Representation of bitmap. + repr: BitmapRepr, + /// Length of this bitmap. Must be `<= MAX_LEN`. + nbits: usize, +} + +impl core::ops::Deref for BitmapVec { + type Target = Bitmap; + + fn deref(&self) -> &Bitmap { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { + // SAFETY: Bitmap is represented inline. + #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] + unsafe { + core::ptr::addr_of!(self.repr.bitmap) + } + } else { + // SAFETY: Bitmap is represented as array of `unsigned long`. + unsafe { self.repr.ptr.as_ptr() } + }; + + // SAFETY: We got the right pointer and invariants of [`Bitmap`] hold. + // An inline bitmap is treated like an array with single element. + unsafe { Bitmap::from_raw(ptr, self.nbits) } + } +} + +impl core::ops::DerefMut for BitmapVec { + fn deref_mut(&mut self) -> &mut Bitmap { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { + // SAFETY: Bitmap is represented inline. + #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] + unsafe { + core::ptr::addr_of_mut!(self.repr.bitmap) + } + } else { + // SAFETY: Bitmap is represented as array of `unsigned long`. + unsafe { self.repr.ptr.as_ptr() } + }; + + // SAFETY: We got the right pointer and invariants of [`BitmapVec`] hold. + // An inline bitmap is treated like an array with single element. + unsafe { Bitmap::from_raw_mut(ptr, self.nbits) } + } +} + +/// Enable ownership transfer to other threads. +/// +/// SAFETY: We own the underlying bitmap representation. +unsafe impl Send for BitmapVec {} + +/// Enable unsynchronized concurrent access to [`BitmapVec`] through shared references. +/// +/// SAFETY: `deref()` will return a reference to a [`Bitmap`]. Its methods +/// take immutable references are either atomic or read-only. +unsafe impl Sync for BitmapVec {} + +impl Drop for BitmapVec { + fn drop(&mut self) { + if self.nbits <= BitmapVec::MAX_INLINE_LEN { + return; + } + // SAFETY: `self.ptr` was returned by the C `bitmap_zalloc`. + // + // INVARIANT: there is no other use of the `self.ptr` after this + // call and the value is being dropped so the broken invariant is + // not observable on function exit. + unsafe { bindings::bitmap_free(self.repr.ptr.as_ptr()) }; + } +} + +impl BitmapVec { + /// The maximum possible length of a `BitmapVec`. + pub const MAX_LEN: usize = i32::MAX as usize; + + /// The maximum length that uses the inline representation. + pub const MAX_INLINE_LEN: usize = usize::BITS as usize; + + /// Construct a longest possible inline [`BitmapVec`]. + #[inline] + pub fn new_inline() -> Self { + // INVARIANT: `nbits <= MAX_INLINE_LEN`, so an inline bitmap is the right repr. + BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits: BitmapVec::MAX_INLINE_LEN, + } + } + + /// Constructs a new [`BitmapVec`]. + /// + /// Fails with [`AllocError`] when the [`BitmapVec`] could not be allocated. This + /// includes the case when `nbits` is greater than `MAX_LEN`. + #[inline] + pub fn new(nbits: usize, flags: Flags) -> Result { + if nbits <= BitmapVec::MAX_INLINE_LEN { + return Ok(BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits, + }); + } + if nbits > Self::MAX_LEN { + return Err(AllocError); + } + let nbits_u32 = u32::try_from(nbits).unwrap(); + // SAFETY: `MAX_INLINE_LEN < nbits` and `nbits <= MAX_LEN`. + let ptr = unsafe { bindings::bitmap_zalloc(nbits_u32, flags.as_raw()) }; + let ptr = NonNull::new(ptr).ok_or(AllocError)?; + // INVARIANT: `ptr` returned by C `bitmap_zalloc` and `nbits` checked. + Ok(BitmapVec { + repr: BitmapRepr { ptr }, + nbits, + }) + } + + /// Returns length of this [`Bitmap`]. + #[allow(clippy::len_without_is_empty)] + #[inline] + pub fn len(&self) -> usize { + self.nbits + } + + /// Fills this `Bitmap` with random bits. + #[cfg(CONFIG_FIND_BIT_BENCHMARK_RUST)] + pub fn fill_random(&mut self) { + // SAFETY: `self.as_mut_ptr` points to either an array of the + // appropriate length or one usize. + unsafe { + bindings::get_random_bytes( + self.as_mut_ptr().cast::(), + usize::div_ceil(self.nbits, bindings::BITS_PER_LONG as usize) + * bindings::BITS_PER_LONG as usize + / 8, + ); + } + } +} + +impl Bitmap { + /// Set bit with index `index`. + /// + /// ATTENTION: `set_bit` is non-atomic, which differs from the naming + /// convention in C code. The corresponding C function is `__set_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.nbits`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.nbits`. + #[inline] + pub fn set_bit(&mut self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: Bit `index` is within bounds. + unsafe { bindings::__set_bit(index, self.as_mut_ptr()) }; + } + + /// Set bit with index `index`, atomically. + /// + /// This is a relaxed atomic operation (no implied memory barriers). + /// + /// ATTENTION: The naming convention differs from C, where the corresponding + /// function is called `set_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn set_bit_atomic(&self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds and the caller has ensured that + // there is no mix of non-atomic and atomic operations. + unsafe { bindings::set_bit(index, self.as_ptr().cast_mut()) }; + } + + /// Clear `index` bit. + /// + /// ATTENTION: `clear_bit` is non-atomic, which differs from the naming + /// convention in C code. The corresponding C function is `__clear_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn clear_bit(&mut self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds. + unsafe { bindings::__clear_bit(index, self.as_mut_ptr()) }; + } + + /// Clear `index` bit, atomically. + /// + /// This is a relaxed atomic operation (no implied memory barriers). + /// + /// ATTENTION: The naming convention differs from C, where the corresponding + /// function is called `clear_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn clear_bit_atomic(&self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds and the caller has ensured that + // there is no mix of non-atomic and atomic operations. + unsafe { bindings::clear_bit(index, self.as_ptr().cast_mut()) }; + } + + /// Copy `src` into this [`Bitmap`] and set any remaining bits to zero. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; + /// use kernel::bitmap::BitmapVec; + /// + /// let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?; + /// + /// assert_eq!(None, long_bitmap.last_bit()); + /// + /// let mut short_bitmap = BitmapVec::new(16, GFP_KERNEL)?; + /// + /// short_bitmap.set_bit(7); + /// long_bitmap.copy_and_extend(&short_bitmap); + /// assert_eq!(Some(7), long_bitmap.last_bit()); + /// + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn copy_and_extend(&mut self, src: &Bitmap) { + let len = core::cmp::min(src.len(), self.len()); + // SAFETY: access to `self` and `src` is within bounds. + unsafe { + bindings::bitmap_copy_and_extend( + self.as_mut_ptr(), + src.as_ptr(), + len as u32, + self.len() as u32, + ) + }; + } + + /// Finds last set bit. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; + /// use kernel::bitmap::BitmapVec; + /// + /// let bitmap = BitmapVec::new(64, GFP_KERNEL)?; + /// + /// match bitmap.last_bit() { + /// Some(idx) => { + /// pr_info!("The last bit has index {idx}.\n"); + /// } + /// None => { + /// pr_info!("All bits in this bitmap are 0.\n"); + /// } + /// } + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn last_bit(&self) -> Option { + // SAFETY: `_find_next_bit` access is within bounds due to invariant. + let index = unsafe { bindings::_find_last_bit(self.as_ptr(), self.len()) }; + if index >= self.len() { + None + } else { + Some(index) + } + } + + /// Finds next set bit, starting from `start`. + /// + /// Returns `None` if `start` is greater or equal to `self.nbits`. + #[inline] + pub fn next_bit(&self, start: usize) -> Option { + bitmap_assert!( + start < self.len(), + "`start` must be < {} was {}", + self.len(), + start + ); + // SAFETY: `_find_next_bit` tolerates out-of-bounds arguments and returns a + // value larger than or equal to `self.len()` in that case. + let index = unsafe { bindings::_find_next_bit(self.as_ptr(), self.len(), start) }; + if index >= self.len() { + None + } else { + Some(index) + } + } + + /// Finds next zero bit, starting from `start`. + /// Returns `None` if `start` is greater than or equal to `self.len()`. + #[inline] + pub fn next_zero_bit(&self, start: usize) -> Option { + bitmap_assert!( + start < self.len(), + "`start` must be < {} was {}", + self.len(), + start + ); + // SAFETY: `_find_next_zero_bit` tolerates out-of-bounds arguments and returns a + // value larger than or equal to `self.len()` in that case. + let index = unsafe { bindings::_find_next_zero_bit(self.as_ptr(), self.len(), start) }; + if index >= self.len() { + None + } else { + Some(index) + } + } +} + +#[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bitmap)] +mod tests { + use super::*; + use kernel::alloc::flags::GFP_KERNEL; + + #[test] + fn bitmap_borrow() { + let fake_bitmap: [usize; 2] = [0, 0]; + let fake_bitmap_len = 2 * usize::BITS as usize; + // SAFETY: `fake_c_bitmap` is an array of expected length. + let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), fake_bitmap_len) }; + assert_eq!(fake_bitmap_len, b.len()); + assert_eq!(None, b.next_bit(0)); + } + + #[test] + fn bitmap_copy() { + let fake_bitmap: usize = 0xFF; + // SAFETY: `fake_c_bitmap` can be used as one-element array of expected length. + let b = unsafe { Bitmap::from_raw(core::ptr::addr_of!(fake_bitmap), 8) }; + assert_eq!(8, b.len()); + assert_eq!(None, b.next_zero_bit(0)); + } + + #[test] + fn bitmap_vec_new() -> Result<(), AllocError> { + let b = BitmapVec::new(0, GFP_KERNEL)?; + assert_eq!(0, b.len()); + + let b = BitmapVec::new(3, GFP_KERNEL)?; + assert_eq!(3, b.len()); + + let b = BitmapVec::new(1024, GFP_KERNEL)?; + assert_eq!(1024, b.len()); + + // Requesting too large values results in [`AllocError`]. + let res = BitmapVec::new(1 << 31, GFP_KERNEL); + assert!(res.is_err()); + Ok(()) + } + + #[test] + fn bitmap_set_clear_find() -> Result<(), AllocError> { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + // Zero-initialized + assert_eq!(None, b.next_bit(0)); + assert_eq!(Some(0), b.next_zero_bit(0)); + assert_eq!(None, b.last_bit()); + + b.set_bit(17); + + assert_eq!(Some(17), b.next_bit(0)); + assert_eq!(Some(17), b.next_bit(17)); + assert_eq!(None, b.next_bit(18)); + assert_eq!(Some(17), b.last_bit()); + + b.set_bit(107); + + assert_eq!(Some(17), b.next_bit(0)); + assert_eq!(Some(17), b.next_bit(17)); + assert_eq!(Some(107), b.next_bit(18)); + assert_eq!(Some(107), b.last_bit()); + + b.clear_bit(17); + + assert_eq!(Some(107), b.next_bit(0)); + assert_eq!(Some(107), b.last_bit()); + Ok(()) + } + + #[test] + fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { + // TODO: Kunit #[test]s do not support `cfg` yet, + // so we add it here in the body. + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + b.set_bit(2048); + b.set_bit_atomic(2048); + b.clear_bit(2048); + b.clear_bit_atomic(2048); + assert_eq!(None, b.next_bit(2048)); + assert_eq!(None, b.next_zero_bit(2048)); + assert_eq!(None, b.last_bit()); + } + Ok(()) + } + + // TODO: uncomment once kunit supports [should_panic] and `cfg`. + // #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + // #[test] + // #[should_panic] + // fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { + // let mut b = BitmapVec::new(128, GFP_KERNEL)?; + // + // b.set_bit(2048); + // } + + #[test] + fn bitmap_copy_and_extend() -> Result<(), AllocError> { + let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?; + + long_bitmap.set_bit(3); + long_bitmap.set_bit(200); + + let mut short_bitmap = BitmapVec::new(32, GFP_KERNEL)?; + + short_bitmap.set_bit(17); + + long_bitmap.copy_and_extend(&short_bitmap); + + // Previous bits have been cleared. + assert_eq!(Some(17), long_bitmap.next_bit(0)); + assert_eq!(Some(17), long_bitmap.last_bit()); + Ok(()) + } +} diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs new file mode 100644 index 000000000..2daead125 --- /dev/null +++ b/rust/kernel/bits.rs @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bit manipulation macros. +//! +//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h) + +use crate::prelude::*; +use core::ops::RangeInclusive; +use macros::paste; + +macro_rules! impl_bit_fn { + ( + $ty:ty + ) => { + paste! { + /// Computes `1 << n` if `n` is in bounds, i.e.: if `n` is smaller than + /// the maximum number of bits supported by the type. + /// + /// Returns [`None`] otherwise. + #[inline] + pub fn [](n: u32) -> Option<$ty> { + (1 as $ty).checked_shl(n) + } + + /// Computes `1 << n` by performing a compile-time assertion that `n` is + /// in bounds. + /// + /// This version is the default and should be used if `n` is known at + /// compile time. + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub const fn [](n: u32) -> $ty { + build_assert!(n < <$ty>::BITS); + (1 as $ty) << n + } + } + }; +} + +impl_bit_fn!(u64); +impl_bit_fn!(u32); +impl_bit_fn!(u16); +impl_bit_fn!(u8); + +macro_rules! impl_genmask_fn { + ( + $ty:ty, + $(#[$genmask_checked_ex:meta])*, + $(#[$genmask_ex:meta])* + ) => { + paste! { + /// Creates a contiguous bitmask for the given range by validating + /// the range at runtime. + /// + /// Returns [`None`] if the range is invalid, i.e.: if the start is + /// greater than the end or if the range is outside of the + /// representable range for the type. + $(#[$genmask_checked_ex])* + #[inline] + pub fn [](range: RangeInclusive) -> Option<$ty> { + let start = *range.start(); + let end = *range.end(); + + if start > end { + return None; + } + + let high = [](end)?; + let low = [](start)?; + Some((high | (high - 1)) & !(low - 1)) + } + + /// Creates a compile-time contiguous bitmask for the given range by + /// performing a compile-time assertion that the range is valid. + /// + /// This version is the default and should be used if the range is known + /// at compile time. + $(#[$genmask_ex])* + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub const fn [](range: RangeInclusive) -> $ty { + let start = *range.start(); + let end = *range.end(); + + build_assert!(start <= end); + + let high = [](end); + let low = [](start); + (high | (high - 1)) & !(low - 1) + } + } + }; +} + +impl_genmask_fn!( + u64, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u64; + /// assert_eq!(genmask_checked_u64(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u64(0..=63), Some(u64::MAX)); + /// assert_eq!(genmask_checked_u64(21..=39), Some(0x0000_00ff_ffe0_0000)); + /// + /// // `80` is out of the supported bit range. + /// assert_eq!(genmask_checked_u64(21..=80), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u64(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u64; + /// assert_eq!(genmask_u64(21..=39), 0x0000_00ff_ffe0_0000); + /// assert_eq!(genmask_u64(0..=0), 0b1); + /// assert_eq!(genmask_u64(0..=63), u64::MAX); + /// ``` +); + +impl_genmask_fn!( + u32, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u32; + /// assert_eq!(genmask_checked_u32(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u32(0..=31), Some(u32::MAX)); + /// assert_eq!(genmask_checked_u32(21..=31), Some(0xffe0_0000)); + /// + /// // `40` is out of the supported bit range. + /// assert_eq!(genmask_checked_u32(21..=40), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u32(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u32; + /// assert_eq!(genmask_u32(21..=31), 0xffe0_0000); + /// assert_eq!(genmask_u32(0..=0), 0b1); + /// assert_eq!(genmask_u32(0..=31), u32::MAX); + /// ``` +); + +impl_genmask_fn!( + u16, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u16; + /// assert_eq!(genmask_checked_u16(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u16(0..=15), Some(u16::MAX)); + /// assert_eq!(genmask_checked_u16(6..=15), Some(0xffc0)); + /// + /// // `20` is out of the supported bit range. + /// assert_eq!(genmask_checked_u16(6..=20), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u16(10..=5), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u16; + /// assert_eq!(genmask_u16(6..=15), 0xffc0); + /// assert_eq!(genmask_u16(0..=0), 0b1); + /// assert_eq!(genmask_u16(0..=15), u16::MAX); + /// ``` +); + +impl_genmask_fn!( + u8, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u8; + /// assert_eq!(genmask_checked_u8(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u8(0..=7), Some(u8::MAX)); + /// assert_eq!(genmask_checked_u8(6..=7), Some(0xc0)); + /// + /// // `10` is out of the supported bit range. + /// assert_eq!(genmask_checked_u8(6..=10), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u8(5..=2), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u8; + /// assert_eq!(genmask_u8(6..=7), 0xc0); + /// assert_eq!(genmask_u8(0..=0), 0b1); + /// assert_eq!(genmask_u8(0..=7), u8::MAX); + /// ``` +); diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs new file mode 100644 index 000000000..32c8d865a --- /dev/null +++ b/rust/kernel/block.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the block layer. + +pub mod mq; + +/// Bit mask for masking out [`SECTOR_SIZE`]. +pub const SECTOR_MASK: u32 = bindings::SECTOR_MASK; + +/// Sectors are size `1 << SECTOR_SHIFT`. +pub const SECTOR_SHIFT: u32 = bindings::SECTOR_SHIFT; + +/// Size of a sector. +pub const SECTOR_SIZE: u32 = bindings::SECTOR_SIZE; + +/// The difference between the size of a page and the size of a sector, +/// expressed as a power of two. +pub const PAGE_SECTORS_SHIFT: u32 = bindings::PAGE_SECTORS_SHIFT; diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs new file mode 100644 index 000000000..1fd0d54dd --- /dev/null +++ b/rust/kernel/block/mq.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides types for implementing block drivers that interface the +//! blk-mq subsystem. +//! +//! To implement a block device driver, a Rust module must do the following: +//! +//! - Implement [`Operations`] for a type `T`. +//! - Create a [`TagSet`]. +//! - Create a [`GenDisk`], via the [`GenDiskBuilder`]. +//! - Add the disk to the system by calling [`GenDiskBuilder::build`] passing in +//! the `TagSet` reference. +//! +//! The types available in this module that have direct C counterparts are: +//! +//! - The [`TagSet`] type that abstracts the C type `struct tag_set`. +//! - The [`GenDisk`] type that abstracts the C type `struct gendisk`. +//! - The [`Request`] type that abstracts the C type `struct request`. +//! +//! The kernel will interface with the block device driver by calling the method +//! implementations of the `Operations` trait. +//! +//! IO requests are passed to the driver as [`kernel::sync::aref::ARef`] +//! instances. The `Request` type is a wrapper around the C `struct request`. +//! The driver must mark end of processing by calling one of the +//! `Request::end`, methods. Failure to do so can lead to deadlock or timeout +//! errors. Please note that the C function `blk_mq_start_request` is implicitly +//! called when the request is queued with the driver. +//! +//! The `TagSet` is responsible for creating and maintaining a mapping between +//! `Request`s and integer ids as well as carrying a pointer to the vtable +//! generated by `Operations`. This mapping is useful for associating +//! completions from hardware with the correct `Request` instance. The `TagSet` +//! determines the maximum queue depth by setting the number of `Request` +//! instances available to the driver, and it determines the number of queues to +//! instantiate for the driver. If possible, a driver should allocate one queue +//! per core, to keep queue data local to a core. +//! +//! One `TagSet` instance can be shared between multiple `GenDisk` instances. +//! This can be useful when implementing drivers where one piece of hardware +//! with one set of IO resources are represented to the user as multiple disks. +//! +//! One significant difference between block device drivers implemented with +//! these Rust abstractions and drivers implemented in C, is that the Rust +//! drivers have to own a reference count on the `Request` type when the IO is +//! in flight. This is to ensure that the C `struct request` instances backing +//! the Rust `Request` instances are live while the Rust driver holds a +//! reference to the `Request`. In addition, the conversion of an integer tag to +//! a `Request` via the `TagSet` would not be sound without this bookkeeping. +//! +//! [`GenDisk`]: gen_disk::GenDisk +//! [`GenDisk`]: gen_disk::GenDisk +//! [`GenDiskBuilder`]: gen_disk::GenDiskBuilder +//! [`GenDiskBuilder::build`]: gen_disk::GenDiskBuilder::build +//! +//! # Examples +//! +//! ```rust +//! use kernel::{ +//! alloc::flags, +//! block::mq::*, +//! new_mutex, +//! prelude::*, +//! sync::{aref::ARef, Arc, Mutex}, +//! }; +//! +//! struct MyBlkDevice; +//! +//! #[vtable] +//! impl Operations for MyBlkDevice { +//! type QueueData = (); +//! +//! fn queue_rq(_queue_data: (), rq: ARef>, _is_last: bool) -> Result { +//! Request::end_ok(rq); +//! Ok(()) +//! } +//! +//! fn commit_rqs(_queue_data: ()) {} +//! +//! fn complete(rq: ARef>) { +//! Request::end_ok(rq) +//! .map_err(|_e| kernel::error::code::EIO) +//! .expect("Fatal error - expected to be able to end request"); +//! } +//! } +//! +//! let tagset: Arc> = +//! Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?; +//! let mut disk = gen_disk::GenDiskBuilder::new() +//! .capacity_sectors(4096) +//! .build(fmt!("myblk"), tagset, ())?; +//! +//! # Ok::<(), kernel::error::Error>(()) +//! ``` + +pub mod gen_disk; +mod operations; +mod request; +mod tag_set; + +pub use operations::Operations; +pub use request::Request; +pub use tag_set::TagSet; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs new file mode 100644 index 000000000..fc97dd873 --- /dev/null +++ b/rust/kernel/block/mq/gen_disk.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic disk abstraction. +//! +//! C header: [`include/linux/blkdev.h`](srctree/include/linux/blkdev.h) +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{Operations, TagSet}, + error::{self, from_err_ptr, Result}, + fmt::{self, Write}, + prelude::*, + static_lock_class, + str::NullTerminatedFormatter, + sync::Arc, + types::{ForeignOwnable, ScopeGuard}, +}; + +/// A builder for [`GenDisk`]. +/// +/// Use this struct to configure and add new [`GenDisk`] to the VFS. +pub struct GenDiskBuilder { + rotational: bool, + logical_block_size: u32, + physical_block_size: u32, + capacity_sectors: u64, +} + +impl Default for GenDiskBuilder { + fn default() -> Self { + Self { + rotational: false, + logical_block_size: bindings::PAGE_SIZE as u32, + physical_block_size: bindings::PAGE_SIZE as u32, + capacity_sectors: 0, + } + } +} + +impl GenDiskBuilder { + /// Create a new instance. + pub fn new() -> Self { + Self::default() + } + + /// Set the rotational media attribute for the device to be built. + pub fn rotational(mut self, rotational: bool) -> Self { + self.rotational = rotational; + self + } + + /// Validate block size by verifying that it is between 512 and `PAGE_SIZE`, + /// and that it is a power of two. + pub fn validate_block_size(size: u32) -> Result { + if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() { + Err(error::code::EINVAL) + } else { + Ok(()) + } + } + + /// Set the logical block size of the device to be built. + /// + /// This method will check that block size is a power of two and between 512 + /// and 4096. If not, an error is returned and the block size is not set. + /// + /// This is the smallest unit the storage device can address. It is + /// typically 4096 bytes. + pub fn logical_block_size(mut self, block_size: u32) -> Result { + Self::validate_block_size(block_size)?; + self.logical_block_size = block_size; + Ok(self) + } + + /// Set the physical block size of the device to be built. + /// + /// This method will check that block size is a power of two and between 512 + /// and 4096. If not, an error is returned and the block size is not set. + /// + /// This is the smallest unit a physical storage device can write + /// atomically. It is usually the same as the logical block size but may be + /// bigger. One example is SATA drives with 4096 byte physical block size + /// that expose a 512 byte logical block size to the operating system. + pub fn physical_block_size(mut self, block_size: u32) -> Result { + Self::validate_block_size(block_size)?; + self.physical_block_size = block_size; + Ok(self) + } + + /// Set the capacity of the device to be built, in sectors (512 bytes). + pub fn capacity_sectors(mut self, capacity: u64) -> Self { + self.capacity_sectors = capacity; + self + } + + /// Build a new `GenDisk` and add it to the VFS. + pub fn build( + self, + name: fmt::Arguments<'_>, + tagset: Arc>, + queue_data: T::QueueData, + ) -> Result> { + let data = queue_data.into_foreign(); + let recover_data = ScopeGuard::new(|| { + // SAFETY: T::QueueData was created by the call to `into_foreign()` above + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + let mut lim: bindings::queue_limits = pin_init::zeroed(); + + lim.logical_block_size = self.logical_block_size; + lim.physical_block_size = self.physical_block_size; + if self.rotational { + lim.features = bindings::BLK_FEAT_ROTATIONAL; + } + + // SAFETY: `tagset.raw_tag_set()` points to a valid and initialized tag set + let gendisk = from_err_ptr(unsafe { + bindings::__blk_mq_alloc_disk( + tagset.raw_tag_set(), + &mut lim, + data, + static_lock_class!().as_ptr(), + ) + })?; + + const TABLE: bindings::block_device_operations = bindings::block_device_operations { + submit_bio: None, + open: None, + release: None, + ioctl: None, + compat_ioctl: None, + check_events: None, + unlock_native_capacity: None, + getgeo: None, + set_read_only: None, + swap_slot_free_notify: None, + report_zones: None, + devnode: None, + alternative_gpt_sector: None, + get_unique_id: None, + // TODO: Set to `THIS_MODULE`. + owner: core::ptr::null_mut(), + pr_ops: core::ptr::null_mut(), + free_disk: None, + poll_bio: None, + }; + + // SAFETY: `gendisk` is a valid pointer as we initialized it above + unsafe { (*gendisk).fops = &TABLE }; + + let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| { + // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and + // has not been added to the VFS on this cleanup path. + unsafe { bindings::put_disk(gendisk) }; + // SAFETY: `data` came from `into_foreign()` above and has not been + // converted back on this cleanup path. + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + // The failure guard now owns both pieces of cleanup; the early guard + // must not run on this path anymore. + recover_data.dismiss(); + + let mut writer = NullTerminatedFormatter::new( + // SAFETY: `gendisk` points to a valid and initialized instance. We + // have exclusive access, since the disk is not added to the VFS + // yet. + unsafe { &mut (*gendisk).disk_name }, + ) + .ok_or(EINVAL)?; + writer.write_fmt(name)?; + + // SAFETY: `gendisk` points to a valid and initialized instance of + // `struct gendisk`. `set_capacity` takes a lock to synchronize this + // operation, so we will not race. + unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) }; + + crate::error::to_result( + // SAFETY: `gendisk` points to a valid and initialized instance of + // `struct gendisk`. + unsafe { + bindings::device_add_disk(core::ptr::null_mut(), gendisk, core::ptr::null_mut()) + }, + )?; + + cleanup_failure.dismiss(); + + // INVARIANT: `gendisk` was initialized above. + // INVARIANT: `gendisk` was added to the VFS via `device_add_disk` above. + // INVARIANT: `gendisk.queue.queue_data` is set to `data` in the call to + // `__blk_mq_alloc_disk` above. + Ok(GenDisk { + _tagset: tagset, + gendisk, + }) + } +} + +/// A generic block device. +/// +/// # Invariants +/// +/// - `gendisk` must always point to an initialized and valid `struct gendisk`. +/// - `gendisk` was added to the VFS through a call to +/// `bindings::device_add_disk`. +/// - `self.gendisk.queue.queuedata` is initialized by a call to `ForeignOwnable::into_foreign`. +pub struct GenDisk { + _tagset: Arc>, + gendisk: *mut bindings::gendisk, +} + +// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a +// `TagSet` It is safe to send this to other threads as long as T is Send. +unsafe impl Send for GenDisk {} + +impl Drop for GenDisk { + fn drop(&mut self) { + // SAFETY: By type invariant of `Self`, `self.gendisk` points to a valid + // and initialized instance of `struct gendisk`, and, `queuedata` was + // initialized with the result of a call to + // `ForeignOwnable::into_foreign`. + let queue_data = unsafe { (*(*self.gendisk).queue).queuedata }; + + // SAFETY: By type invariant, `self.gendisk` points to a valid and + // initialized instance of `struct gendisk`, and it was previously added + // to the VFS. + unsafe { bindings::del_gendisk(self.gendisk) }; + + // SAFETY: By type invariant, `self.gendisk` was added to the VFS, so + // `put_disk()` must follow `del_gendisk()` to drop the final gendisk + // reference and trigger the remaining release path. + unsafe { bindings::put_disk(self.gendisk) }; + + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with + // a call to `ForeignOwnable::into_foreign` to create `queuedata`. + // `ForeignOwnable::from_foreign` is only called here. + drop(unsafe { T::QueueData::from_foreign(queue_data) }); + } +} diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs new file mode 100644 index 000000000..861903e18 --- /dev/null +++ b/rust/kernel/block/mq/operations.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides an interface for blk-mq drivers to implement. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{request::RequestDataWrapper, Request}, + error::{from_result, Result}, + prelude::*, + sync::{aref::ARef, Refcount}, + types::ForeignOwnable, +}; +use core::marker::PhantomData; + +type ForeignBorrowed<'a, T> = ::Borrowed<'a>; + +/// Implement this trait to interface blk-mq as block devices. +/// +/// To implement a block device driver, implement this trait as described in the +/// [module level documentation]. The kernel will use the implementation of the +/// functions defined in this trait to interface a block device driver. Note: +/// There is no need for an exit_request() implementation, because the `drop` +/// implementation of the [`Request`] type will be invoked by automatically by +/// the C/Rust glue logic. +/// +/// [module level documentation]: kernel::block::mq +#[macros::vtable] +pub trait Operations: Sized { + /// Data associated with the `struct request_queue` that is allocated for + /// the `GenDisk` associated with this `Operations` implementation. + type QueueData: ForeignOwnable; + + /// Called by the kernel to queue a request with the driver. If `is_last` is + /// `false`, the driver is allowed to defer committing the request. + fn queue_rq( + queue_data: ForeignBorrowed<'_, Self::QueueData>, + rq: ARef>, + is_last: bool, + ) -> Result; + + /// Called by the kernel to indicate that queued requests should be submitted. + fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>); + + /// Called by the kernel when the request is completed. + fn complete(rq: ARef>); + + /// Called by the kernel to poll the device for completed requests. Only + /// used for poll queues. + fn poll() -> bool { + build_error!(crate::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A vtable for blk-mq to interact with a block device driver. +/// +/// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern +/// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`. +/// +/// For general documentation of these methods, see the kernel source +/// documentation related to `struct blk_mq_operations` in +/// [`include/linux/blk-mq.h`]. +/// +/// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h +pub(crate) struct OperationsVTable(PhantomData); + +impl OperationsVTable { + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - The caller of this function must ensure that the pointee of `bd` is + /// valid for reads for the duration of this function. + /// - This function must be called for an initialized and live `hctx`. That + /// is, `Self::init_hctx_callback` was called and + /// `Self::exit_hctx_callback()` was not yet called. + /// - `(*bd).rq` must point to an initialized and live `bindings:request`. + /// That is, `Self::init_request_callback` was called but + /// `Self::exit_request_callback` was not yet called for the request. + /// - `(*bd).rq` must be owned by the driver. That is, the block layer must + /// promise to not access the request until the driver calls + /// `bindings::blk_mq_end_request` for the request. + unsafe extern "C" fn queue_rq_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + bd: *const bindings::blk_mq_queue_data, + ) -> bindings::blk_status_t { + // SAFETY: `bd.rq` is valid as required by the safety requirement for + // this function. + let request = unsafe { &*(*bd).rq.cast::>() }; + + // One refcount for the ARef, one for being in flight + request.wrapper_ref().refcount().set(2); + + // SAFETY: + // - We own a refcount that we took above. We pass that to `ARef`. + // - By the safety requirements of this function, `request` is a valid + // `struct request` and the private data is properly initialized. + // - `rq` will be alive until `blk_mq_end_request` is called and is + // reference counted by `ARef` until then. + let rq = unsafe { Request::aref_from_raw((*bd).rq) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with + // a call to `ForeignOwnable::into_foreign` to create `queuedata`. + // `ForeignOwnable::from_foreign` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + + // SAFETY: We have exclusive access and we just set the refcount above. + unsafe { Request::start_unchecked(&rq) }; + + let ret = T::queue_rq( + queue_data, + rq, + // SAFETY: `bd` is valid as required by the safety requirement for + // this function. + unsafe { (*bd).last }, + ); + + if let Err(e) = ret { + e.to_blk_status() + } else { + bindings::BLK_STS_OK as bindings::blk_status_t + } + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. The caller + /// must ensure that `hctx` is valid. + unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) { + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a + // call to `ForeignOwnable::into_foreign()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + T::commit_rqs(queue_data) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. `rq` must + /// point to a valid request that has been marked as completed. The pointee + /// of `rq` must be valid for write for the duration of this function. + unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { + // SAFETY: This function can only be dispatched through + // `Request::complete`. We leaked a refcount then which we pick back up + // now. + let aref = unsafe { Request::aref_from_raw(rq) }; + T::complete(aref); + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. + unsafe extern "C" fn poll_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _iob: *mut bindings::io_comp_batch, + ) -> crate::ffi::c_int { + T::poll().into() + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. This + /// function may only be called once before `exit_hctx_callback` is called + /// for the same context. + unsafe extern "C" fn init_hctx_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _tagset_data: *mut crate::ffi::c_void, + _hctx_idx: crate::ffi::c_uint, + ) -> crate::ffi::c_int { + from_result(|| Ok(0)) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. + unsafe extern "C" fn exit_hctx_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _hctx_idx: crate::ffi::c_uint, + ) { + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - This function may only be called by blk-mq C infrastructure. + /// - `_set` must point to an initialized `TagSet`. + /// - `rq` must point to an initialized `bindings::request`. + /// - The allocation pointed to by `rq` must be at the size of `Request` + /// plus the size of `RequestDataWrapper`. + unsafe extern "C" fn init_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: crate::ffi::c_uint, + _numa_node: crate::ffi::c_int, + ) -> crate::ffi::c_int { + from_result(|| { + // SAFETY: By the safety requirements of this function, `rq` points + // to a valid allocation. + let pdu = unsafe { Request::wrapper_ptr(rq.cast::>()) }; + + // SAFETY: The refcount field is allocated but not initialized, so + // it is valid for writes. + unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; + + Ok(0) + }) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - This function may only be called by blk-mq C infrastructure. + /// - `_set` must point to an initialized `TagSet`. + /// - `rq` must point to an initialized and valid `Request`. + unsafe extern "C" fn exit_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: crate::ffi::c_uint, + ) { + // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory + // for the request data. + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::(); + + // SAFETY: `pdu` is valid for read and write and is properly initialised. + unsafe { core::ptr::drop_in_place(pdu) }; + } + + const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops { + queue_rq: Some(Self::queue_rq_callback), + queue_rqs: None, + commit_rqs: Some(Self::commit_rqs_callback), + get_budget: None, + put_budget: None, + set_rq_budget_token: None, + get_rq_budget_token: None, + timeout: None, + poll: if T::HAS_POLL { + Some(Self::poll_callback) + } else { + None + }, + complete: Some(Self::complete_callback), + init_hctx: Some(Self::init_hctx_callback), + exit_hctx: Some(Self::exit_hctx_callback), + init_request: Some(Self::init_request_callback), + exit_request: Some(Self::exit_request_callback), + cleanup_rq: None, + busy: None, + map_queues: None, + #[cfg(CONFIG_BLK_DEBUG_FS)] + show_rq: None, + }; + + pub(crate) const fn build() -> &'static bindings::blk_mq_ops { + &Self::VTABLE + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs new file mode 100644 index 000000000..ce3e30c81 --- /dev/null +++ b/rust/kernel/block/mq/request.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides a wrapper for the C `struct request` type. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::Operations, + error::Result, + sync::{ + aref::{ARef, AlwaysRefCounted}, + atomic::Relaxed, + Refcount, + }, + types::Opaque, +}; +use core::{marker::PhantomData, ptr::NonNull}; + +/// A wrapper around a blk-mq [`struct request`]. This represents an IO request. +/// +/// # Implementation details +/// +/// There are four states for a request that the Rust bindings care about: +/// +/// 1. Request is owned by block layer (refcount 0). +/// 2. Request is owned by driver but with zero [`ARef`]s in existence +/// (refcount 1). +/// 3. Request is owned by driver with exactly one [`ARef`] in existence +/// (refcount 2). +/// 4. Request is owned by driver with more than one [`ARef`] in existence +/// (refcount > 2). +/// +/// +/// We need to track 1 and 2 to ensure we fail tag to request conversions for +/// requests that are not owned by the driver. +/// +/// We need to track 3 and 4 to ensure that it is safe to end the request and hand +/// back ownership to the block layer. +/// +/// Note that the driver can still obtain new `ARef` even if there is no `ARef`s in existence by +/// using `tag_to_rq`, hence the need to distinguish B and C. +/// +/// The states are tracked through the private `refcount` field of +/// `RequestDataWrapper`. This structure lives in the private data area of the C +/// [`struct request`]. +/// +/// # Invariants +/// +/// * `self.0` is a valid [`struct request`] created by the C portion of the +/// kernel. +/// * The private data area associated with this request must be an initialized +/// and valid `RequestDataWrapper`. +/// * `self` is reference counted by atomic modification of +/// `self.wrapper_ref().refcount()`. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h +/// +#[repr(transparent)] +pub struct Request(Opaque, PhantomData); + +impl Request { + /// Create an [`ARef`] from a [`struct request`] pointer. + /// + /// # Safety + /// + /// * The caller must own a refcount on `ptr` that is transferred to the + /// returned [`ARef`]. + /// * The type invariants for [`Request`] must hold for the pointee of `ptr`. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h + pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef { + // INVARIANT: By the safety requirements of this function, invariants are upheld. + // SAFETY: By the safety requirement of this function, we own a + // reference count that we can pass to `ARef`. + unsafe { ARef::from_raw(NonNull::new_unchecked(ptr.cast())) } + } + + /// Notify the block layer that a request is going to be processed now. + /// + /// The block layer uses this hook to do proper initializations such as + /// starting the timeout timer. It is a requirement that block device + /// drivers call this function when starting to process a request. + /// + /// # Safety + /// + /// The caller must have exclusive ownership of `self`, that is + /// `self.wrapper_ref().refcount() == 2`. + pub(crate) unsafe fn start_unchecked(this: &ARef) { + // SAFETY: By type invariant, `self.0` is a valid `struct request` and + // we have exclusive access. + unsafe { bindings::blk_mq_start_request(this.0.get()) }; + } + + /// Try to take exclusive ownership of `this` by dropping the refcount to 0. + /// This fails if `this` is not the only [`ARef`] pointing to the underlying + /// [`Request`]. + /// + /// If the operation is successful, [`Ok`] is returned with a pointer to the + /// C [`struct request`]. If the operation fails, `this` is returned in the + /// [`Err`] variant. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h + fn try_set_end(this: ARef) -> Result<*mut bindings::request, ARef> { + // To hand back the ownership, we need the current refcount to be 2. + // Since we can race with `TagSet::tag_to_rq`, this needs to atomically reduce + // refcount to 0. `Refcount` does not provide a way to do this, so use the underlying + // atomics directly. + if let Err(_old) = this + .wrapper_ref() + .refcount() + .as_atomic() + .cmpxchg(2, 0, Relaxed) + { + return Err(this); + } + + let request_ptr = this.0.get(); + core::mem::forget(this); + + Ok(request_ptr) + } + + /// Notify the block layer that the request has been completed without errors. + /// + /// This function will return [`Err`] if `this` is not the only [`ARef`] + /// referencing the request. + pub fn end_ok(this: ARef) -> Result<(), ARef> { + let request_ptr = Self::try_set_end(this)?; + + // SAFETY: By type invariant, `this.0` was a valid `struct request`. The + // success of the call to `try_set_end` guarantees that there are no + // `ARef`s pointing to this request. Therefore it is safe to hand it + // back to the block layer. + unsafe { + bindings::blk_mq_end_request( + request_ptr, + bindings::BLK_STS_OK as bindings::blk_status_t, + ) + }; + + Ok(()) + } + + /// Complete the request by scheduling `Operations::complete` for + /// execution. + /// + /// The function may be scheduled locally, via SoftIRQ or remotely via IPMI. + /// See `blk_mq_complete_request_remote` in [`blk-mq.c`] for details. + /// + /// [`blk-mq.c`]: srctree/block/blk-mq.c + pub fn complete(this: ARef) { + let ptr = ARef::into_raw(this).cast::().as_ptr(); + // SAFETY: By type invariant, `self.0` is a valid `struct request` + if !unsafe { bindings::blk_mq_complete_request_remote(ptr) } { + // SAFETY: We released a refcount above that we can reclaim here. + let this = unsafe { Request::aref_from_raw(ptr) }; + T::complete(this); + } + } + + /// Return a pointer to the [`RequestDataWrapper`] stored in the private area + /// of the request structure. + /// + /// # Safety + /// + /// - `this` must point to a valid allocation of size at least size of + /// [`Self`] plus size of [`RequestDataWrapper`]. + pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull { + let request_ptr = this.cast::(); + // SAFETY: By safety requirements for this function, `this` is a + // valid allocation. + let wrapper_ptr = + unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::() }; + // SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation + // and is not null. + unsafe { NonNull::new_unchecked(wrapper_ptr) } + } + + /// Return a reference to the [`RequestDataWrapper`] stored in the private + /// area of the request structure. + pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { + // SAFETY: By type invariant, `self.0` is a valid allocation. Further, + // the private data associated with this request is initialized and + // valid. The existence of `&self` guarantees that the private data is + // valid as a shared reference. + unsafe { Self::wrapper_ptr(core::ptr::from_ref(self).cast_mut()).as_ref() } + } +} + +/// A wrapper around data stored in the private area of the C [`struct request`]. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h +pub(crate) struct RequestDataWrapper { + /// The Rust request refcount has the following states: + /// + /// - 0: The request is owned by C block layer. + /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. + /// - 2+: There are [`ARef`] references to the request. + refcount: Refcount, +} + +impl RequestDataWrapper { + /// Return a reference to the refcount of the request that is embedding + /// `self`. + pub(crate) fn refcount(&self) -> &Refcount { + &self.refcount + } + + /// Return a pointer to the refcount of the request that is embedding the + /// pointee of `this`. + /// + /// # Safety + /// + /// - `this` must point to a live allocation of at least the size of `Self`. + pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { + // SAFETY: Because of the safety requirements of this function, the + // field projection is safe. + unsafe { &raw mut (*this).refcount } + } +} + +// SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut +// self` methods and `&self` methods that mutate `self` are internally +// synchronized. +unsafe impl Send for Request {} + +// SAFETY: Shared access is thread-safe for `Request`. `&self` methods that +// mutate `self` are internally synchronized` +unsafe impl Sync for Request {} + +// SAFETY: All instances of `Request` are reference counted. This +// implementation of `AlwaysRefCounted` ensure that increments to the ref count +// keeps the object alive in memory at least until a matching reference count +// decrement is executed. +unsafe impl AlwaysRefCounted for Request { + fn inc_ref(&self) { + self.wrapper_ref().refcount().inc(); + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: The type invariants of `ARef` guarantee that `obj` is valid + // for read. + let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() }; + // SAFETY: The type invariant of `Request` guarantees that the private + // data area is initialized and valid. + let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) }; + + #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))] + let is_zero = refcount.dec_and_test(); + + #[cfg(CONFIG_DEBUG_MISC)] + if is_zero { + panic!("Request reached refcount zero in Rust abstractions"); + } + } +} diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs new file mode 100644 index 000000000..dae9df408 --- /dev/null +++ b/rust/kernel/block/mq/tag_set.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides the `TagSet` struct to wrap the C `struct blk_mq_tag_set`. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use core::pin::Pin; + +use crate::{ + bindings, + block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, + error::{self, Result}, + prelude::try_pin_init, + types::Opaque, +}; +use core::{convert::TryInto, marker::PhantomData}; +use pin_init::{pin_data, pinned_drop, PinInit}; + +/// A wrapper for the C `struct blk_mq_tag_set`. +/// +/// `struct blk_mq_tag_set` contains a `struct list_head` and so must be pinned. +/// +/// # Invariants +/// +/// - `inner` is initialized and valid. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct TagSet { + #[pin] + inner: Opaque, + _p: PhantomData, +} + +impl TagSet { + /// Try to create a new tag set + pub fn new( + nr_hw_queues: u32, + num_tags: u32, + num_maps: u32, + ) -> impl PinInit { + let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed(); + let tag_set: Result<_> = core::mem::size_of::() + .try_into() + .map(|cmd_size| { + bindings::blk_mq_tag_set { + ops: OperationsVTable::::build(), + nr_hw_queues, + timeout: 0, // 0 means default which is 30Hz in C + numa_node: bindings::NUMA_NO_NODE, + queue_depth: num_tags, + cmd_size, + flags: 0, + driver_data: core::ptr::null_mut::(), + nr_maps: num_maps, + ..tag_set + } + }) + .map(Opaque::new) + .map_err(|e| e.into()); + + try_pin_init!(TagSet { + inner <- tag_set.pin_chain(|tag_set| { + // SAFETY: we do not move out of `tag_set`. + let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) }; + // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`. + error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) + }), + _p: PhantomData, + }) + } + + /// Return the pointer to the wrapped `struct blk_mq_tag_set` + pub(crate) fn raw_tag_set(&self) -> *mut bindings::blk_mq_tag_set { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for TagSet { + fn drop(self: Pin<&mut Self>) { + // SAFETY: By type invariant `inner` is valid and has been properly + // initialized during construction. + unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) }; + } +} diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs new file mode 100644 index 000000000..3566f0234 --- /dev/null +++ b/rust/kernel/bug.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024, 2025 FUJITA Tomonori + +//! Support for BUG and WARN functionality. +//! +//! C header: [`include/asm-generic/bug.h`](srctree/include/asm-generic/bug.h) + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(CONFIG_DEBUG_BUGVERBOSE)] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + const _FILE: &[u8] = $file.as_bytes(); + // Plus one for null-terminator. + static FILE: [u8; _FILE.len() + 1] = { + let mut bytes = [0; _FILE.len() + 1]; + let mut i = 0; + while i < _FILE.len() { + bytes[i] = _FILE[i]; + i += 1; + } + bytes + }; + + // SAFETY: + // - `file`, `line`, `flags`, and `size` are all compile-time constants or + // symbols, preventing any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + file = sym FILE, + line = const line!(), + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + + if false { + _ = $file; + } + + // SAFETY: + // - `flags` and `size` are all compile-time constants, preventing + // any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, CONFIG_UML))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + + // SAFETY: It is always safe to call `warn_slowpath_fmt()` + // with a valid null-terminated string. + unsafe { + $crate::bindings::warn_slowpath_fmt( + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), + line!() as $crate::ffi::c_int, + $flags as $crate::ffi::c_uint, + ::core::ptr::null(), + ); + } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + + // SAFETY: It is always safe to call `WARN_ON()`. + unsafe { $crate::bindings::WARN_ON(true) } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(any(testlib, not(CONFIG_BUG)))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; +} + +#[doc(hidden)] +pub const fn bugflag_taint(value: u32) -> u32 { + value << 8 +} + +/// Report a warning if `cond` is true and return the condition's evaluation result. +#[macro_export] +macro_rules! warn_on { + ($cond:expr) => {{ + let cond = $cond; + + #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] + const COND_STR: &str = file!(); + + if cond { + const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); + + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); + } + cond + }}; +} diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs new file mode 100644 index 000000000..c3acb9b68 --- /dev/null +++ b/rust/kernel/build_assert.rs @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Various assertions that happen during build-time. +//! +//! There are three types of build-time assertions that you can use: +//! - [`static_assert!`] +//! - [`const_assert!`] +//! - [`build_assert!`] +//! +//! The ones towards the bottom of the list are more expressive, while the ones towards the top of +//! the list are more robust and trigger earlier in the compilation pipeline. Therefore, you should +//! prefer the ones towards the top of the list wherever possible. +//! +//! # Choosing the correct assertion +//! +//! If you're asserting outside any bodies (e.g. initializers or function bodies), you should use +//! [`static_assert!`] as it is the only assertion that can be used in that context. +//! +//! Inside bodies, if your assertion condition does not depend on any variable or generics, you +//! should use [`static_assert!`]. If the condition depends on generics, but not variables +//! (including function arguments), you should use [`const_assert!`]. Otherwise, use +//! [`build_assert!`]. The same is true regardless if the function is `const fn`. +//! +//! ``` +//! // Outside any bodies. +//! static_assert!(core::mem::size_of::() == 1); +//! // `const_assert!` and `build_assert!` cannot be used here, they will fail to compile. +//! +//! #[inline(always)] +//! fn foo(v: usize) { +//! static_assert!(core::mem::size_of::() == 1); // Preferred. +//! const_assert!(core::mem::size_of::() == 1); // Discouraged. +//! build_assert!(core::mem::size_of::() == 1); // Discouraged. +//! +//! // `static_assert!(N > 1);` is not allowed. +//! const_assert!(N > 1); // Preferred. +//! build_assert!(N > 1); // Discouraged. +//! +//! // `static_assert!(v > 1);` is not allowed. +//! // `const_assert!(v > 1);` is not allowed. +//! build_assert!(v > 1); // Works. +//! } +//! ``` +//! +//! # Detailed behavior +//! +//! `static_assert!()` is equivalent to `static_assert` in C. It requires `expr` to be a constant +//! expression. This expression cannot refer to any generics. A `static_assert!(expr)` in a program +//! is always evaluated, regardless if the function it appears in is used or not. This is also the +//! only usable assertion outside a body. +//! +//! `const_assert!()` has no direct C equivalence. It is a more powerful version of +//! `static_assert!()`, where it may refer to generics in a function. Note that due to the ability +//! to refer to generics, the assertion is tied to a specific instance of a function. So if it is +//! used in a generic function that is not instantiated, the assertion will not be checked. For this +//! reason, `static_assert!()` is preferred wherever possible. +//! +//! `build_assert!()` is equivalent to `BUILD_BUG_ON`. It is even more powerful than +//! `const_assert!()` because it can be used to check tautologies that depend on runtime value (this +//! is the same as `BUILD_BUG_ON`). However, the assertion failure mechanism can possibly be +//! undefined symbols and linker errors, it is not developer friendly to debug, so it is recommended +//! to avoid it and prefer other two assertions where possible. + +#[doc(inline)] +pub use crate::{ + build_assert_macro as build_assert, + build_error, + const_assert, + static_assert, // +}; + +#[doc(hidden)] +pub use build_error::build_error as build_error_fn; + +/// Static assert (i.e. compile-time assert). +/// +/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`]. +/// +/// An optional panic message can be supplied after the expression. +/// Currently only a string literal without formatting is supported +/// due to constness limitations of the [`assert!`] macro. +/// +/// The feature may be added to Rust in the future: see [RFC 2790]. +/// +/// You cannot refer to generics or variables with [`static_assert!`]. If you need to refer to +/// generics, use [`const_assert!`]; if you need to refer to variables, use [`build_assert!`]. See +/// the [module documentation](self). +/// +/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert +/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert +/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790 +/// +/// # Examples +/// +/// ``` +/// static_assert!(42 > 24); +/// static_assert!(core::mem::size_of::() == 1); +/// +/// const X: &[u8] = b"bar"; +/// static_assert!(X[1] == b'a'); +/// +/// const fn f(x: i32) -> i32 { +/// x + 2 +/// } +/// static_assert!(f(40) == 42); +/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input."); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! static_assert { + ($condition:expr $(,$arg:literal)?) => { + const _: () = ::core::assert!($condition $(,$arg)?); + }; +} + +/// Assertion during constant evaluation. +/// +/// This is a more powerful version of [`static_assert!`] that can refer to generics inside +/// functions or implementation blocks. However, it also has a limitation where it can only appear +/// in places where statements can appear; for example, you cannot use it as an item in the module. +/// +/// [`static_assert!`] should be preferred if no generics are referred to in the condition. You +/// cannot refer to variables with [`const_assert!`] (even inside `const fn`); if you need the +/// capability, use [`build_assert!`]. See the [module documentation](self). +/// +/// # Examples +/// +/// ``` +/// fn foo() { +/// const_assert!(N > 1); +/// } +/// +/// fn bar() { +/// const_assert!(size_of::() > 0, "T cannot be ZST"); +/// } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! const_assert { + ($condition:expr $(,$arg:literal)?) => { + const { ::core::assert!($condition $(,$arg)?) }; + }; +} + +/// Fails the build if the code path calling `build_error!` can possibly be executed. +/// +/// If the macro is executed in const context, `build_error!` will panic. +/// If the compiler or optimizer cannot guarantee that `build_error!` can never +/// be called, a build error will be triggered. +/// +/// # Examples +/// +/// ``` +/// #[inline] +/// fn foo(a: usize) -> usize { +/// a.checked_add(1).unwrap_or_else(|| build_error!("overflow")) +/// } +/// +/// assert_eq!(foo(usize::MAX - 1), usize::MAX); // OK. +/// // foo(usize::MAX); // Fails to compile. +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! build_error { + () => {{ + $crate::build_assert::build_error_fn("") + }}; + ($msg:expr) => {{ + $crate::build_assert::build_error_fn($msg) + }}; +} + +/// Asserts that a boolean expression is `true` at compile time. +/// +/// If the condition is evaluated to `false` in const context, `build_assert!` +/// will panic. If the compiler or optimizer cannot guarantee the condition will +/// be evaluated to `true`, a build error will be triggered. +/// +/// When a condition depends on a function argument, the function must be annotated with +/// `#[inline(always)]`. Without this attribute, the compiler may choose to not inline the +/// function, preventing it from optimizing out the error path. +/// +/// If the assertion condition does not depend on any variables or generics, you should use +/// [`static_assert!`]. If the assertion condition does not depend on variables, but does depend on +/// generics, you should use [`const_assert!`]. See the [module documentation](self). +/// +/// # Examples +/// +/// ``` +/// #[inline(always)] // Important. +/// fn bar(n: usize) { +/// build_assert!(n > 1); +/// } +/// +/// fn foo() { +/// bar(2); +/// } +/// +/// #[inline(always)] // Important. +/// const fn const_bar(n: usize) { +/// build_assert!(n > 1); +/// } +/// +/// const _: () = const_bar(2); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! build_assert_macro { + ($cond:expr $(,)?) => {{ + if !$cond { + $crate::build_assert::build_error_fn(concat!("assertion failed: ", stringify!($cond))); + } + }}; + ($cond:expr, $msg:expr) => {{ + if !$cond { + $crate::build_assert::build_error_fn($msg); + } + }}; +} diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs new file mode 100644 index 000000000..7abbd0767 --- /dev/null +++ b/rust/kernel/clk.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Clock abstractions. +//! +//! C header: [`include/linux/clk.h`](srctree/include/linux/clk.h) +//! +//! Reference: + +use crate::ffi::c_ulong; + +/// The frequency unit. +/// +/// Represents a frequency in hertz, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::clk::Hertz; +/// +/// let hz = 1_000_000_000; +/// let rate = Hertz(hz); +/// +/// assert_eq!(rate.as_hz(), hz); +/// assert_eq!(rate, Hertz(hz)); +/// assert_eq!(rate, Hertz::from_khz(hz / 1_000)); +/// assert_eq!(rate, Hertz::from_mhz(hz / 1_000_000)); +/// assert_eq!(rate, Hertz::from_ghz(hz / 1_000_000_000)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct Hertz(pub c_ulong); + +impl Hertz { + const KHZ_TO_HZ: c_ulong = 1_000; + const MHZ_TO_HZ: c_ulong = 1_000_000; + const GHZ_TO_HZ: c_ulong = 1_000_000_000; + + /// Create a new instance from kilohertz (kHz) + pub const fn from_khz(khz: c_ulong) -> Self { + Self(khz * Self::KHZ_TO_HZ) + } + + /// Create a new instance from megahertz (MHz) + pub const fn from_mhz(mhz: c_ulong) -> Self { + Self(mhz * Self::MHZ_TO_HZ) + } + + /// Create a new instance from gigahertz (GHz) + pub const fn from_ghz(ghz: c_ulong) -> Self { + Self(ghz * Self::GHZ_TO_HZ) + } + + /// Get the frequency in hertz + pub const fn as_hz(&self) -> c_ulong { + self.0 + } + + /// Get the frequency in kilohertz + pub const fn as_khz(&self) -> c_ulong { + self.0 / Self::KHZ_TO_HZ + } + + /// Get the frequency in megahertz + pub const fn as_mhz(&self) -> c_ulong { + self.0 / Self::MHZ_TO_HZ + } + + /// Get the frequency in gigahertz + pub const fn as_ghz(&self) -> c_ulong { + self.0 / Self::GHZ_TO_HZ + } +} + +impl From for c_ulong { + fn from(freq: Hertz) -> Self { + freq.0 + } +} + +#[cfg(CONFIG_COMMON_CLK)] +mod common_clk { + use super::Hertz; + use crate::{ + device::Device, + error::{from_err_ptr, to_result, Result}, + prelude::*, + }; + + use core::{ops::Deref, ptr}; + + /// A reference-counted clock. + /// + /// Rust abstraction for the C [`struct clk`]. + /// + /// # Invariants + /// + /// A [`Clk`] instance holds either a pointer to a valid [`struct clk`] created by the C + /// portion of the kernel or a `NULL` pointer. + /// + /// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`Clk`]. + /// + /// # Examples + /// + /// The following example demonstrates how to obtain and configure a clock for a device. + /// + /// ``` + /// use kernel::clk::{Clk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = Clk::get(dev, Some(c"apb_clk"))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + #[repr(transparent)] + pub struct Clk(*mut bindings::clk); + + // SAFETY: It is safe to call `clk_put` on another thread than where `clk_get` was called. + unsafe impl Send for Clk {} + + // SAFETY: It is safe to call any combination of the `&self` methods in parallel, as the + // methods are synchronized internally. + unsafe impl Sync for Clk {} + + impl Clk { + /// Gets [`Clk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get`] API. + /// + /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); + + // SAFETY: It is safe to call [`clk_get`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`Clk`] goes out of scope. + Ok(Self(from_err_ptr(unsafe { + bindings::clk_get(dev.as_raw(), con_id) + })?)) + } + + /// Obtain the raw [`struct clk`] pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::clk { + self.0 + } + + /// Enable the clock. + /// + /// Equivalent to the kernel's [`clk_enable`] API. + /// + /// [`clk_enable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_enable + #[inline] + pub fn enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_enable`]. + to_result(unsafe { bindings::clk_enable(self.as_raw()) }) + } + + /// Disable the clock. + /// + /// Equivalent to the kernel's [`clk_disable`] API. + /// + /// [`clk_disable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_disable + #[inline] + pub fn disable(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable`]. + unsafe { bindings::clk_disable(self.as_raw()) }; + } + + /// Prepare the clock. + /// + /// Equivalent to the kernel's [`clk_prepare`] API. + /// + /// [`clk_prepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_prepare + #[inline] + pub fn prepare(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare`]. + to_result(unsafe { bindings::clk_prepare(self.as_raw()) }) + } + + /// Unprepare the clock. + /// + /// Equivalent to the kernel's [`clk_unprepare`] API. + /// + /// [`clk_unprepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_unprepare + #[inline] + pub fn unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_unprepare`]. + unsafe { bindings::clk_unprepare(self.as_raw()) }; + } + + /// Prepare and enable the clock. + /// + /// Equivalent to calling [`Clk::prepare`] followed by [`Clk::enable`]. + #[inline] + pub fn prepare_enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare_enable`]. + to_result(unsafe { bindings::clk_prepare_enable(self.as_raw()) }) + } + + /// Disable and unprepare the clock. + /// + /// Equivalent to calling [`Clk::disable`] followed by [`Clk::unprepare`]. + #[inline] + pub fn disable_unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable_unprepare`]. + unsafe { bindings::clk_disable_unprepare(self.as_raw()) }; + } + + /// Get clock's rate. + /// + /// Equivalent to the kernel's [`clk_get_rate`] API. + /// + /// [`clk_get_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_rate + #[inline] + pub fn rate(&self) -> Hertz { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_get_rate`]. + Hertz(unsafe { bindings::clk_get_rate(self.as_raw()) }) + } + + /// Set clock's rate. + /// + /// Equivalent to the kernel's [`clk_set_rate`] API. + /// + /// [`clk_set_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_set_rate + #[inline] + pub fn set_rate(&self, rate: Hertz) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_set_rate`]. + to_result(unsafe { bindings::clk_set_rate(self.as_raw(), rate.as_hz()) }) + } + } + + impl Drop for Clk { + fn drop(&mut self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for [`clk_put`]. + unsafe { bindings::clk_put(self.as_raw()) }; + } + } + + /// A reference-counted optional clock. + /// + /// A lightweight wrapper around an optional [`Clk`]. An [`OptionalClk`] represents a [`Clk`] + /// that a driver can function without but may improve performance or enable additional + /// features when available. + /// + /// # Invariants + /// + /// An [`OptionalClk`] instance encapsulates a [`Clk`] with either a valid [`struct clk`] or + /// `NULL` pointer. + /// + /// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`OptionalClk`]. + /// + /// # Examples + /// + /// The following example demonstrates how to obtain and configure an optional clock for a + /// device. The code functions correctly whether or not the clock is available. + /// + /// ``` + /// use kernel::clk::{OptionalClk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = OptionalClk::get(dev, Some(c"apb_clk"))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + pub struct OptionalClk(Clk); + + impl OptionalClk { + /// Gets [`OptionalClk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get_optional`] API. + /// + /// [`clk_get_optional`]: + /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); + + // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`OptionalClk`] goes out of + // scope. + Ok(Self(Clk(from_err_ptr(unsafe { + bindings::clk_get_optional(dev.as_raw(), con_id) + })?))) + } + } + + // Make [`OptionalClk`] behave like [`Clk`]. + impl Deref for OptionalClk { + type Target = Clk; + + fn deref(&self) -> &Clk { + &self.0 + } + } +} + +#[cfg(CONFIG_COMMON_CLK)] +pub use common_clk::*; diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs new file mode 100644 index 000000000..cd082b83e --- /dev/null +++ b/rust/kernel/configfs.rs @@ -0,0 +1,1047 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! configfs interface: Userspace-driven Kernel Object Configuration +//! +//! configfs is an in-memory pseudo file system for configuration of kernel +//! modules. Please see the [C documentation] for details and intended use of +//! configfs. +//! +//! This module does not support the following configfs features: +//! +//! - Items. All group children are groups. +//! - Symlink support. +//! - `disconnect_notify` hook. +//! - Default groups. +//! +//! See the [`rust_configfs.rs`] sample for a full example use of this module. +//! +//! C header: [`include/linux/configfs.h`](srctree/include/linux/configfs.h) +//! +//! # Examples +//! +//! ```ignore +//! use kernel::alloc::flags; +//! use kernel::configfs_attrs; +//! use kernel::configfs; +//! use kernel::new_mutex; +//! use kernel::page::PAGE_SIZE; +//! use kernel::sync::Mutex; +//! use kernel::ThisModule; +//! +//! #[pin_data] +//! struct RustConfigfs { +//! #[pin] +//! config: configfs::Subsystem, +//! } +//! +//! impl kernel::InPlaceModule for RustConfigfs { +//! fn init(_module: &'static ThisModule) -> impl PinInit { +//! pr_info!("Rust configfs sample (init)\n"); +//! +//! let item_type = configfs_attrs! { +//! container: configfs::Subsystem, +//! data: Configuration, +//! attributes: [ +//! message: 0, +//! bar: 1, +//! ], +//! }; +//! +//! try_pin_init!(Self { +//! config <- configfs::Subsystem::new( +//! c"rust_configfs", item_type, Configuration::new() +//! ), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct Configuration { +//! message: &'static CStr, +//! #[pin] +//! bar: Mutex<(KBox<[u8; PAGE_SIZE]>, usize)>, +//! } +//! +//! impl Configuration { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! message: c"Hello World\n", +//! bar <- new_mutex!((KBox::new([0; PAGE_SIZE], flags::GFP_KERNEL)?, 0)), +//! }) +//! } +//! } +//! +//! #[vtable] +//! impl configfs::AttributeOperations<0> for Configuration { +//! type Data = Configuration; +//! +//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result { +//! pr_info!("Show message\n"); +//! let data = container.message; +//! page[0..data.len()].copy_from_slice(data); +//! Ok(data.len()) +//! } +//! } +//! +//! #[vtable] +//! impl configfs::AttributeOperations<1> for Configuration { +//! type Data = Configuration; +//! +//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result { +//! pr_info!("Show bar\n"); +//! let guard = container.bar.lock(); +//! let data = guard.0.as_slice(); +//! let len = guard.1; +//! page[0..len].copy_from_slice(&data[0..len]); +//! Ok(len) +//! } +//! +//! fn store(container: &Configuration, page: &[u8]) -> Result { +//! pr_info!("Store bar\n"); +//! let mut guard = container.bar.lock(); +//! guard.0[0..page.len()].copy_from_slice(page); +//! guard.1 = page.len(); +//! Ok(()) +//! } +//! } +//! ``` +//! +//! [C documentation]: srctree/Documentation/filesystems/configfs.rst +//! [`rust_configfs.rs`]: srctree/samples/rust/rust_configfs.rs + +use crate::alloc::flags; +use crate::container_of; +use crate::page::PAGE_SIZE; +use crate::prelude::*; +use crate::str::CString; +use crate::sync::Arc; +use crate::sync::ArcBorrow; +use crate::types::Opaque; +use core::cell::UnsafeCell; +use core::marker::PhantomData; + +/// A configfs subsystem. +/// +/// This is the top level entrypoint for a configfs hierarchy. To register +/// with configfs, embed a field of this type into your kernel module struct. +#[pin_data(PinnedDrop)] +pub struct Subsystem { + #[pin] + subsystem: Opaque, + #[pin] + data: Data, +} + +// SAFETY: We do not provide any operations on `Subsystem`. +unsafe impl Sync for Subsystem {} + +// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads. +unsafe impl Send for Subsystem {} + +impl Subsystem { + /// Create an initializer for a [`Subsystem`]. + /// + /// The subsystem will appear in configfs as a directory name given by + /// `name`. The attributes available in directory are specified by + /// `item_type`. + pub fn new( + name: &'static CStr, + item_type: &'static ItemType, Data>, + data: impl PinInit, + ) -> impl PinInit { + try_pin_init!(Self { + subsystem <- pin_init::init_zeroed().chain( + |place: &mut Opaque| { + // SAFETY: We initialized the required fields of `place.group` above. + unsafe { + bindings::config_group_init_type_name( + &mut (*place.get()).su_group, + name.as_char_ptr(), + item_type.as_ptr(), + ) + }; + + // SAFETY: `place.su_mutex` is valid for use as a mutex. + unsafe { + bindings::__mutex_init( + &mut (*place.get()).su_mutex, + kernel::optional_name!().as_char_ptr(), + kernel::static_lock_class!().as_ptr(), + ) + } + Ok(()) + } + ), + data <- data, + }) + .pin_chain(|this| { + crate::error::to_result( + // SAFETY: We initialized `this.subsystem` according to C API contract above. + unsafe { bindings::configfs_register_subsystem(this.subsystem.get()) }, + ) + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Subsystem { + fn drop(self: Pin<&mut Self>) { + // SAFETY: We registered `self.subsystem` in the initializer returned by `Self::new`. + unsafe { bindings::configfs_unregister_subsystem(self.subsystem.get()) }; + // SAFETY: We initialized the mutex in `Subsystem::new`. + unsafe { bindings::mutex_destroy(&raw mut (*self.subsystem.get()).su_mutex) }; + } +} + +/// Trait that allows offset calculations for structs that embed a +/// `bindings::config_group`. +/// +/// Users of the configfs API should not need to implement this trait. +/// +/// # Safety +/// +/// - Implementers of this trait must embed a `bindings::config_group`. +/// - Methods must be implemented according to method documentation. +pub unsafe trait HasGroup { + /// Return the address of the `bindings::config_group` embedded in [`Self`]. + /// + /// # Safety + /// + /// - `this` must be a valid allocation of at least the size of [`Self`]. + unsafe fn group(this: *const Self) -> *const bindings::config_group; + + /// Return the address of the [`Self`] that `group` is embedded in. + /// + /// # Safety + /// + /// - `group` must point to the `bindings::config_group` that is embedded in + /// [`Self`]. + unsafe fn container_of(group: *const bindings::config_group) -> *const Self; +} + +// SAFETY: `Subsystem` embeds a field of type `bindings::config_group` +// within the `subsystem` field. +unsafe impl HasGroup for Subsystem { + unsafe fn group(this: *const Self) -> *const bindings::config_group { + // SAFETY: By impl and function safety requirement this projection is in bounds. + unsafe { &raw const (*(*this).subsystem.get()).su_group } + } + + unsafe fn container_of(group: *const bindings::config_group) -> *const Self { + // SAFETY: By impl and function safety requirement this projection is in bounds. + let c_subsys_ptr = unsafe { container_of!(group, bindings::configfs_subsystem, su_group) }; + let opaque_ptr = c_subsys_ptr.cast::>(); + // SAFETY: By impl and function safety requirement, `opaque_ptr` and the + // pointer it returns, are within the same allocation. + unsafe { container_of!(opaque_ptr, Subsystem, subsystem) } + } +} + +/// A configfs group. +/// +/// To add a subgroup to configfs, pass this type as `ctype` to +/// [`crate::configfs_attrs`] when creating a group in [`GroupOperations::make_group`]. +#[pin_data] +pub struct Group { + #[pin] + group: Opaque, + #[pin] + data: Data, +} + +impl Group { + /// Create an initializer for a new group. + /// + /// When instantiated, the group will appear as a directory with the name + /// given by `name` and it will contain attributes specified by `item_type`. + pub fn new( + name: CString, + item_type: &'static ItemType, Data>, + data: impl PinInit, + ) -> impl PinInit { + try_pin_init!(Self { + group <- pin_init::init_zeroed().chain(|v: &mut Opaque| { + let place = v.get(); + let name = name.to_bytes_with_nul().as_ptr(); + // SAFETY: It is safe to initialize a group once it has been zeroed. + unsafe { + bindings::config_group_init_type_name(place, name.cast(), item_type.as_ptr()) + }; + Ok(()) + }), + data <- data, + }) + } +} + +// SAFETY: `Group` embeds a field of type `bindings::config_group` +// within the `group` field. +unsafe impl HasGroup for Group { + unsafe fn group(this: *const Self) -> *const bindings::config_group { + Opaque::cast_into( + // SAFETY: By impl and function safety requirements this field + // projection is within bounds of the allocation. + unsafe { &raw const (*this).group }, + ) + } + + unsafe fn container_of(group: *const bindings::config_group) -> *const Self { + let opaque_ptr = group.cast::>(); + // SAFETY: By impl and function safety requirement, `opaque_ptr` and + // pointer it returns will be in the same allocation. + unsafe { container_of!(opaque_ptr, Self, group) } + } +} + +/// # Safety +/// +/// `this` must be a valid pointer. +/// +/// If `this` does not represent the root group of a configfs subsystem, +/// `this` must be a pointer to a `bindings::config_group` embedded in a +/// `Group`. +/// +/// Otherwise, `this` must be a pointer to a `bindings::config_group` that +/// is embedded in a `bindings::configfs_subsystem` that is embedded in a +/// `Subsystem`. +unsafe fn get_group_data<'a, Parent>(this: *mut bindings::config_group) -> &'a Parent { + // SAFETY: `this` is a valid pointer. + let is_root = unsafe { (*this).cg_subsys.is_null() }; + + if !is_root { + // SAFETY: By C API contact,`this` was returned from a call to + // `make_group`. The pointer is known to be embedded within a + // `Group`. + unsafe { &(*Group::::container_of(this)).data } + } else { + // SAFETY: By C API contract, `this` is a pointer to the + // `bindings::config_group` field within a `Subsystem`. + unsafe { &(*Subsystem::container_of(this)).data } + } +} + +struct GroupOperationsVTable(PhantomData<(Parent, Child)>); + +impl GroupOperationsVTable +where + Parent: GroupOperations, + Child: 'static, +{ + /// # Safety + /// + /// `this` must be a valid pointer. + /// + /// If `this` does not represent the root group of a configfs subsystem, + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// Otherwise, `this` must be a pointer to a `bindings::config_group` that + /// is embedded in a `bindings::configfs_subsystem` that is embedded in a + /// `Subsystem`. + /// + /// `name` must point to a null terminated string. + unsafe extern "C" fn make_group( + this: *mut bindings::config_group, + name: *const kernel::ffi::c_char, + ) -> *mut bindings::config_group { + // SAFETY: By function safety requirements of this function, this call + // is safe. + let parent_data = unsafe { get_group_data(this) }; + + let group_init = match Parent::make_group( + parent_data, + // SAFETY: By function safety requirements, name points to a null + // terminated string. + unsafe { CStr::from_char_ptr(name) }, + ) { + Ok(init) => init, + Err(e) => return e.to_ptr(), + }; + + let child_group = > as InPlaceInit>>::try_pin_init( + group_init, + flags::GFP_KERNEL, + ); + + match child_group { + Ok(child_group) => { + let child_group_ptr = child_group.into_raw(); + // SAFETY: We allocated the pointee of `child_ptr` above as a + // `Group`. + unsafe { Group::::group(child_group_ptr) }.cast_mut() + } + Err(e) => e.to_ptr(), + } + } + + /// # Safety + /// + /// If `this` does not represent the root group of a configfs subsystem, + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// Otherwise, `this` must be a pointer to a `bindings::config_group` that + /// is embedded in a `bindings::configfs_subsystem` that is embedded in a + /// `Subsystem`. + /// + /// `item` must point to a `bindings::config_item` within a + /// `bindings::config_group` within a `Group`. + unsafe extern "C" fn drop_item( + this: *mut bindings::config_group, + item: *mut bindings::config_item, + ) { + // SAFETY: By function safety requirements of this function, this call + // is safe. + let parent_data = unsafe { get_group_data(this) }; + + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + let c_child_group_ptr = unsafe { container_of!(item, bindings::config_group, cg_item) }; + // SAFETY: By function safety requirements, `c_child_group_ptr` is + // embedded within a `Group`. + let r_child_group_ptr = unsafe { Group::::container_of(c_child_group_ptr) }; + + if Parent::HAS_DROP_ITEM { + // SAFETY: We called `into_raw` to produce `r_child_group_ptr` in + // `make_group`. + let arc: Arc> = unsafe { Arc::from_raw(r_child_group_ptr.cast_mut()) }; + + Parent::drop_item(parent_data, arc.as_arc_borrow()); + arc.into_raw(); + } + + // SAFETY: By C API contract, we are required to drop a refcount on + // `item`. + unsafe { bindings::config_item_put(item) }; + } + + const VTABLE: bindings::configfs_group_operations = bindings::configfs_group_operations { + make_item: None, + make_group: Some(Self::make_group), + disconnect_notify: None, + drop_item: Some(Self::drop_item), + is_visible: None, + is_bin_visible: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_group_operations { + &Self::VTABLE + } +} + +struct ItemOperationsVTable(PhantomData<(Container, Data)>); + +impl ItemOperationsVTable, Data> +where + Data: 'static, +{ + /// # Safety + /// + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// This function will destroy the pointee of `this`. The pointee of `this` + /// must not be accessed after the function returns. + unsafe extern "C" fn release(this: *mut bindings::config_item) { + // SAFETY: By function safety requirements, `this` is embedded in a + // `config_group`. + let c_group_ptr = unsafe { kernel::container_of!(this, bindings::config_group, cg_item) }; + // SAFETY: By function safety requirements, `c_group_ptr` is + // embedded within a `Group`. + let r_group_ptr = unsafe { Group::::container_of(c_group_ptr) }; + + // SAFETY: We called `into_raw` on `r_group_ptr` in + // `make_group`. + let pin_self: Arc> = unsafe { Arc::from_raw(r_group_ptr.cast_mut()) }; + drop(pin_self); + } + + const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations { + release: Some(Self::release), + allow_link: None, + drop_link: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_item_operations { + &Self::VTABLE + } +} + +impl ItemOperationsVTable, Data> { + const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations { + release: None, + allow_link: None, + drop_link: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_item_operations { + &Self::VTABLE + } +} + +/// Operations implemented by configfs groups that can create subgroups. +/// +/// Implement this trait on structs that embed a [`Subsystem`] or a [`Group`]. +#[vtable] +pub trait GroupOperations { + /// The child data object type. + /// + /// This group will create subgroups (subdirectories) backed by this kind of + /// object. + type Child: 'static; + + /// Creates a new subgroup. + /// + /// The kernel will call this method in response to `mkdir(2)` in the + /// directory representing `this`. + /// + /// To accept the request to create a group, implementations should + /// return an initializer of a `Group`. To prevent creation, + /// return a suitable error. + fn make_group(&self, name: &CStr) -> Result, Error>>; + + /// Prepares the group for removal from configfs. + /// + /// The kernel will call this method before the directory representing `_child` is removed from + /// configfs. + /// + /// Implementations can use this method to do house keeping before configfs drops its + /// reference to `Child`. + /// + /// NOTE: "drop" in the name of this function is not related to the Rust drop term. Rather, the + /// name is inherited from the callback name in the underlying C code. + fn drop_item(&self, _child: ArcBorrow<'_, Group>) { + kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A configfs attribute. +/// +/// An attribute appears as a file in configfs, inside a folder that represent +/// the group that the attribute belongs to. +#[repr(transparent)] +pub struct Attribute { + attribute: Opaque, + _p: PhantomData<(O, Data)>, +} + +// SAFETY: We do not provide any operations on `Attribute`. +unsafe impl Sync for Attribute {} + +// SAFETY: Ownership of `Attribute` can safely be transferred to other threads. +unsafe impl Send for Attribute {} + +impl Attribute +where + O: AttributeOperations, +{ + /// # Safety + /// + /// `item` must be embedded in a `bindings::config_group`. + /// + /// If `item` does not represent the root group of a configfs subsystem, + /// the group must be embedded in a `Group`. + /// + /// Otherwise, the group must be a embedded in a + /// `bindings::configfs_subsystem` that is embedded in a `Subsystem`. + /// + /// `page` must point to a writable buffer of size at least [`PAGE_SIZE`]. + unsafe extern "C" fn show( + item: *mut bindings::config_item, + page: *mut kernel::ffi::c_char, + ) -> isize { + let c_group: *mut bindings::config_group = + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + unsafe { container_of!(item, bindings::config_group, cg_item) }; + + // SAFETY: The function safety requirements for this function satisfy + // the conditions for this call. + let data: &Data = unsafe { get_group_data(c_group) }; + + // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`. + let ret = O::show(data, unsafe { &mut *(page.cast::<[u8; PAGE_SIZE]>()) }); + + match ret { + Ok(size) => size as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `item` must be embedded in a `bindings::config_group`. + /// + /// If `item` does not represent the root group of a configfs subsystem, + /// the group must be embedded in a `Group`. + /// + /// Otherwise, the group must be a embedded in a + /// `bindings::configfs_subsystem` that is embedded in a `Subsystem`. + /// + /// `page` must point to a readable buffer of size at least `size`. + unsafe extern "C" fn store( + item: *mut bindings::config_item, + page: *const kernel::ffi::c_char, + size: usize, + ) -> isize { + let c_group: *mut bindings::config_group = + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + unsafe { container_of!(item, bindings::config_group, cg_item) }; + + // SAFETY: The function safety requirements for this function satisfy + // the conditions for this call. + let data: &Data = unsafe { get_group_data(c_group) }; + + let ret = O::store( + data, + // SAFETY: By function safety requirements, `page` is readable + // for at least `size`. + unsafe { core::slice::from_raw_parts(page.cast(), size) }, + ); + + match ret { + Ok(()) => size as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// Create a new attribute. + /// + /// The attribute will appear as a file with name given by `name`. + pub const fn new(name: &'static CStr) -> Self { + Self { + attribute: Opaque::new(bindings::configfs_attribute { + ca_name: crate::str::as_char_ptr_in_const_context(name), + ca_owner: core::ptr::null_mut(), + ca_mode: 0o660, + show: Some(Self::show), + store: if O::HAS_STORE { + Some(Self::store) + } else { + None + }, + }), + _p: PhantomData, + } + } +} + +/// Operations supported by an attribute. +/// +/// Implement this trait on type and pass that type as generic parameter when +/// creating an [`Attribute`]. The type carrying the implementation serve no +/// purpose other than specifying the attribute operations. +/// +/// This trait must be implemented on the `Data` type of for types that +/// implement `HasGroup`. The trait must be implemented once for each +/// attribute of the group. The constant type parameter `ID` maps the +/// implementation to a specific `Attribute`. `ID` must be passed when declaring +/// attributes via the [`kernel::configfs_attrs`] macro, to tie +/// `AttributeOperations` implementations to concrete named attributes. +#[vtable] +pub trait AttributeOperations { + /// The type of the object that contains the field that is backing the + /// attribute for this operation. + type Data; + + /// Renders the value of an attribute. + /// + /// This function is called by the kernel to read the value of an attribute. + /// + /// Implementations should write the rendering of the attribute to `page` + /// and return the number of bytes written. + fn show(data: &Self::Data, page: &mut [u8; PAGE_SIZE]) -> Result; + + /// Stores the value of an attribute. + /// + /// This function is called by the kernel to update the value of an attribute. + /// + /// Implementations should parse the value from `page` and update internal + /// state to reflect the parsed value. + fn store(_data: &Self::Data, _page: &[u8]) -> Result { + kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A list of attributes. +/// +/// This type is used to construct a new [`ItemType`]. It represents a list of +/// [`Attribute`] that will appear in the directory representing a [`Group`]. +/// Users should not directly instantiate this type, rather they should use the +/// [`kernel::configfs_attrs`] macro to declare a static set of attributes for a +/// group. +/// +/// # Note +/// +/// Instances of this type are constructed statically at compile by the +/// [`kernel::configfs_attrs`] macro. +#[repr(transparent)] +pub struct AttributeList( + /// Null terminated Array of pointers to [`Attribute`]. The type is [`c_void`] + /// to conform to the C API. + UnsafeCell<[*mut kernel::ffi::c_void; N]>, + PhantomData, +); + +// SAFETY: Ownership of `AttributeList` can safely be transferred to other threads. +unsafe impl Send for AttributeList {} + +// SAFETY: We do not provide any operations on `AttributeList` that need synchronization. +unsafe impl Sync for AttributeList {} + +impl AttributeList { + /// # Safety + /// + /// This function must only be called by the [`kernel::configfs_attrs`] + /// macro. + #[doc(hidden)] + pub const unsafe fn new() -> Self { + Self(UnsafeCell::new([core::ptr::null_mut(); N]), PhantomData) + } + + /// # Safety + /// + /// The caller must ensure that there are no other concurrent accesses to + /// `self`. That is, the caller has exclusive access to `self.` + #[doc(hidden)] + pub const unsafe fn add( + &'static self, + attribute: &'static Attribute, + ) where + O: AttributeOperations, + { + // We need a space at the end of our list for a null terminator. + const { assert!(I < N - 1, "Invalid attribute index") }; + + // SAFETY: By function safety requirements, we have exclusive access to + // `self` and the reference created below will be exclusive. + unsafe { (&mut *self.0.get())[I] = core::ptr::from_ref(attribute).cast_mut().cast() }; + } +} + +/// A representation of the attributes that will appear in a [`Group`] or +/// [`Subsystem`]. +/// +/// Users should not directly instantiate objects of this type. Rather, they +/// should use the [`kernel::configfs_attrs`] macro to statically declare the +/// shape of a [`Group`] or [`Subsystem`]. +#[pin_data] +pub struct ItemType { + #[pin] + item_type: Opaque, + _p: PhantomData<(Container, Data)>, +} + +// SAFETY: We do not provide any operations on `ItemType` that need synchronization. +unsafe impl Sync for ItemType {} + +// SAFETY: Ownership of `ItemType` can safely be transferred to other threads. +unsafe impl Send for ItemType {} + +macro_rules! impl_item_type { + ($tpe:ty) => { + impl ItemType<$tpe, Data> { + #[doc(hidden)] + pub const fn new_with_child_ctor( + owner: &'static ThisModule, + attributes: &'static AttributeList, + ) -> Self + where + Data: GroupOperations, + Child: 'static, + { + Self { + item_type: Opaque::new(bindings::config_item_type { + ct_owner: owner.as_ptr(), + ct_group_ops: GroupOperationsVTable::::vtable_ptr().cast_mut(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), + ct_bin_attrs: core::ptr::null_mut(), + }), + _p: PhantomData, + } + } + + #[doc(hidden)] + pub const fn new( + owner: &'static ThisModule, + attributes: &'static AttributeList, + ) -> Self { + Self { + item_type: Opaque::new(bindings::config_item_type { + ct_owner: owner.as_ptr(), + ct_group_ops: core::ptr::null_mut(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), + ct_bin_attrs: core::ptr::null_mut(), + }), + _p: PhantomData, + } + } + } + }; +} + +impl_item_type!(Subsystem); +impl_item_type!(Group); + +impl ItemType { + fn as_ptr(&self) -> *const bindings::config_item_type { + self.item_type.get() + } +} + +/// Define a list of configfs attributes statically. +/// +/// Invoking the macro in the following manner: +/// +/// ```ignore +/// let item_type = configfs_attrs! { +/// container: configfs::Subsystem, +/// data: Configuration, +/// child: Child, +/// attributes: [ +/// message: 0, +/// bar: 1, +/// ], +/// }; +/// ``` +/// +/// Expands the following output: +/// +/// ```ignore +/// let item_type = { +/// static CONFIGURATION_MESSAGE_ATTR: kernel::configfs::Attribute< +/// 0, +/// Configuration, +/// Configuration, +/// > = unsafe { +/// kernel::configfs::Attribute::new({ +/// const S: &str = "message\u{0}"; +/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul( +/// S.as_bytes() +/// ) { +/// Ok(v) => v, +/// Err(_) => { +/// core::panicking::panic_fmt(core::const_format_args!( +/// "string contains interior NUL" +/// )); +/// } +/// }; +/// C +/// }) +/// }; +/// +/// static CONFIGURATION_BAR_ATTR: kernel::configfs::Attribute< +/// 1, +/// Configuration, +/// Configuration +/// > = unsafe { +/// kernel::configfs::Attribute::new({ +/// const S: &str = "bar\u{0}"; +/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul( +/// S.as_bytes() +/// ) { +/// Ok(v) => v, +/// Err(_) => { +/// core::panicking::panic_fmt(core::const_format_args!( +/// "string contains interior NUL" +/// )); +/// } +/// }; +/// C +/// }) +/// }; +/// +/// const N: usize = (1usize + (1usize + 0usize)) + 1usize; +/// +/// static CONFIGURATION_ATTRS: kernel::configfs::AttributeList = +/// unsafe { kernel::configfs::AttributeList::new() }; +/// +/// { +/// const N: usize = 0usize; +/// unsafe { CONFIGURATION_ATTRS.add::(&CONFIGURATION_MESSAGE_ATTR) }; +/// } +/// +/// { +/// const N: usize = (1usize + 0usize); +/// unsafe { CONFIGURATION_ATTRS.add::(&CONFIGURATION_BAR_ATTR) }; +/// } +/// +/// static CONFIGURATION_TPE: +/// kernel::configfs::ItemType ,Configuration> +/// = kernel::configfs::ItemType::< +/// configfs::Subsystem, +/// Configuration +/// >::new_with_child_ctor::( +/// ::kernel::module::this_module::(), +/// &CONFIGURATION_ATTRS +/// ); +/// +/// &CONFIGURATION_TPE +/// } +/// ``` +#[allow(clippy::crate_in_macro_def)] +#[macro_export] +macro_rules! configfs_attrs { + ( + container: $container:ty, + data: $data:ty, + attributes: [ + $($name:ident: $attr:literal),* $(,)? + ] $(,)? + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child(), + @no_child(x), + @attrs($($name $attr)*), + @eat($($name $attr,)*), + @assign(), + @cnt(0usize), + ) + }; + ( + container: $container:ty, + data: $data:ty, + child: $child:ty, + attributes: [ + $($name:ident: $attr:literal),* $(,)? + ] $(,)? + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child($child), + @no_child(), + @attrs($($name $attr)*), + @eat($($name $attr,)*), + @assign(), + @cnt(0usize), + ) + }; + (count: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($aname:ident $aattr:literal)*), + @eat($name:ident $attr:literal, $($rname:ident $rattr:literal,)*), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child($($child)?), + @no_child($($no_child)?), + @attrs($($aname $aattr)*), + @eat($($rname $rattr,)*), + @assign($($assign)* { + const N: usize = $cnt; + // The following macro text expands to a call to `Attribute::add`. + + // SAFETY: By design of this macro, the name of the variable we + // invoke the `add` method on below, is not visible outside of + // the macro expansion. The macro does not operate concurrently + // on this variable, and thus we have exclusive access to the + // variable. + unsafe { + $crate::macros::paste!( + [< $data:upper _ATTRS >] + .add::(&[< $data:upper _ $name:upper _ATTR >]) + ) + }; + }), + @cnt(1usize + $cnt), + ) + }; + (count: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($aname:ident $aattr:literal)*), + @eat(), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => + { + $crate::configfs_attrs!( + final: + @container($container), + @data($data), + @child($($child)?), + @no_child($($no_child)?), + @attrs($($aname $aattr)*), + @assign($($assign)*), + @cnt($cnt), + ) + }; + (final: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($name:ident $attr:literal)*), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => + { + $crate::macros::paste!{ + { + $( + // SAFETY: We are expanding `configfs_attrs`. + static [< $data:upper _ $name:upper _ATTR >]: + $crate::configfs::Attribute<$attr, $data, $data> = + unsafe { + $crate::configfs::Attribute::new( + $crate::c_str!(::core::stringify!($name)), + ) + }; + )* + + + // We need space for a null terminator. + const N: usize = $cnt + 1usize; + + // SAFETY: We are expanding `configfs_attrs`. + static [< $data:upper _ATTRS >]: + $crate::configfs::AttributeList = + unsafe { $crate::configfs::AttributeList::new() }; + + $($assign)* + + $( + const [<$no_child:upper>]: bool = true; + + static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> = + $crate::configfs::ItemType::<$container, $data>::new::( + $crate::module::this_module::(), + &[<$ data:upper _ATTRS >] + ); + )? + + $( + static [< $data:upper _TPE >]: + $crate::configfs::ItemType<$container, $data> = + $crate::configfs::ItemType::<$container, $data>:: + new_with_child_ctor::( + $crate::module::this_module::(), + &[<$ data:upper _ATTRS >] + ); + )? + + & [< $data:upper _TPE >] + } + } + }; + +} + +pub use crate::configfs_attrs; diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs new file mode 100644 index 000000000..cb6c0338e --- /dev/null +++ b/rust/kernel/cpu.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic CPU definitions. +//! +//! C header: [`include/linux/cpu.h`](srctree/include/linux/cpu.h) + +use crate::{bindings, device::Device, error::Result, prelude::ENODEV}; + +/// Returns the maximum number of possible CPUs in the current system configuration. +#[inline] +pub fn nr_cpu_ids() -> u32 { + #[cfg(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS))] + { + bindings::NR_CPUS + } + + #[cfg(not(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS)))] + // SAFETY: `nr_cpu_ids` is a valid global provided by the kernel. + unsafe { + bindings::nr_cpu_ids + } +} + +/// The CPU ID. +/// +/// Represents a CPU identifier as a wrapper around an [`u32`]. +/// +/// # Invariants +/// +/// The CPU ID lies within the range `[0, nr_cpu_ids())`. +/// +/// # Examples +/// +/// ``` +/// use kernel::cpu::CpuId; +/// +/// let cpu = 0; +/// +/// // SAFETY: 0 is always a valid CPU number. +/// let id = unsafe { CpuId::from_u32_unchecked(cpu) }; +/// +/// assert_eq!(id.as_u32(), cpu); +/// assert!(CpuId::from_i32(0).is_some()); +/// assert!(CpuId::from_i32(-1).is_none()); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct CpuId(u32); + +impl CpuId { + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_i32_unchecked(id: i32) -> Self { + debug_assert!(id >= 0); + debug_assert!((id as u32) < nr_cpu_ids()); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id as u32) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_i32(id: i32) -> Option { + if id < 0 || id as u32 >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id as u32)) + } + } + + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_u32_unchecked(id: u32) -> Self { + debug_assert!(id < nr_cpu_ids()); + + // Ensure the `id` fits in an [`i32`] as it's also representable that way. + debug_assert!(id <= i32::MAX as u32); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_u32(id: u32) -> Option { + if id >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id)) + } + } + + /// Returns CPU number. + #[inline] + pub fn as_u32(&self) -> u32 { + self.0 + } + + /// Returns the ID of the CPU the code is currently running on. + /// + /// The returned value is considered unstable because it may change + /// unexpectedly due to preemption or CPU migration. It should only be + /// used when the context ensures that the task remains on the same CPU + /// or the users could use a stale (yet valid) CPU ID. + #[inline] + pub fn current() -> Self { + // SAFETY: raw_smp_processor_id() always returns a valid CPU ID. + unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) } + } +} + +impl From for u32 { + fn from(id: CpuId) -> Self { + id.as_u32() + } +} + +impl From for i32 { + fn from(id: CpuId) -> Self { + id.as_u32() as i32 + } +} + +/// Creates a new instance of CPU's device. +/// +/// # Safety +/// +/// Reference counting is not implemented for the CPU device in the C code. When a CPU is +/// hot-unplugged, the corresponding CPU device is unregistered, but its associated memory +/// is not freed. +/// +/// Callers must ensure that the CPU device is not used after it has been unregistered. +/// This can be achieved, for example, by registering a CPU hotplug notifier and removing +/// any references to the CPU device within the notifier's callback. +pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> { + // SAFETY: It is safe to call `get_cpu_device()` for any CPU. + let ptr = unsafe { bindings::get_cpu_device(u32::from(cpu)) }; + if ptr.is_null() { + return Err(ENODEV); + } + + // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to + // a `struct device` and is never freed by the C code. + Ok(unsafe { Device::from_raw(ptr) }) +} diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs new file mode 100644 index 000000000..affa2b949 --- /dev/null +++ b/rust/kernel/cpufreq.rs @@ -0,0 +1,1416 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU frequency scaling. +//! +//! This module provides rust abstractions for interacting with the cpufreq subsystem. +//! +//! C header: [`include/linux/cpufreq.h`](srctree/include/linux/cpufreq.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + cpu::CpuId, + cpumask, + device::{Bound, Device}, + devres, + error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + ffi::{c_char, c_ulong}, + prelude::*, + types::ForeignOwnable, + types::Opaque, +}; + +#[cfg(CONFIG_COMMON_CLK)] +use crate::clk::Clk; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + ops::{Deref, DerefMut}, + pin::Pin, + ptr, +}; + +use macros::vtable; + +/// Maximum length of CPU frequency driver's name. +const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize; + +/// Default transition latency value in nanoseconds. +pub const DEFAULT_TRANSITION_LATENCY_NS: u32 = bindings::CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + +/// CPU frequency driver flags. +pub mod flags { + /// Driver needs to update internal limits even if frequency remains unchanged. + pub const NEED_UPDATE_LIMITS: u16 = 1 << 0; + + /// Platform where constants like `loops_per_jiffy` are unaffected by frequency changes. + pub const CONST_LOOPS: u16 = 1 << 1; + + /// Register driver as a thermal cooling device automatically. + pub const IS_COOLING_DEV: u16 = 1 << 2; + + /// Supports multiple clock domains with per-policy governors in `cpu/cpuN/cpufreq/`. + pub const HAVE_GOVERNOR_PER_POLICY: u16 = 1 << 3; + + /// Allows post-change notifications outside of the `target()` routine. + pub const ASYNC_NOTIFICATION: u16 = 1 << 4; + + /// Ensure CPU starts at a valid frequency from the driver's freq-table. + pub const NEED_INITIAL_FREQ_CHECK: u16 = 1 << 5; + + /// Disallow governors with `dynamic_switching` capability. + pub const NO_AUTO_DYNAMIC_SWITCHING: u16 = 1 << 6; +} + +/// Relations from the C code. +const CPUFREQ_RELATION_L: u32 = 0; +const CPUFREQ_RELATION_H: u32 = 1; +const CPUFREQ_RELATION_C: u32 = 2; + +/// Can be used with any of the above values. +const CPUFREQ_RELATION_E: u32 = 1 << 2; + +/// CPU frequency selection relations. +/// +/// CPU frequency selection relations, each optionally marked as "efficient". +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Relation { + /// Select the lowest frequency at or above target. + Low(bool), + /// Select the highest frequency below or at target. + High(bool), + /// Select the closest frequency to the target. + Close(bool), +} + +impl Relation { + // Construct from a C-compatible `u32` value. + fn new(val: u32) -> Result { + let efficient = val & CPUFREQ_RELATION_E != 0; + + Ok(match val & !CPUFREQ_RELATION_E { + CPUFREQ_RELATION_L => Self::Low(efficient), + CPUFREQ_RELATION_H => Self::High(efficient), + CPUFREQ_RELATION_C => Self::Close(efficient), + _ => return Err(EINVAL), + }) + } +} + +impl From for u32 { + // Convert to a C-compatible `u32` value. + fn from(rel: Relation) -> Self { + let (mut val, efficient) = match rel { + Relation::Low(e) => (CPUFREQ_RELATION_L, e), + Relation::High(e) => (CPUFREQ_RELATION_H, e), + Relation::Close(e) => (CPUFREQ_RELATION_C, e), + }; + + if efficient { + val |= CPUFREQ_RELATION_E; + } + + val + } +} + +/// Policy data. +/// +/// Rust abstraction for the C `struct cpufreq_policy_data`. +/// +/// # Invariants +/// +/// A [`PolicyData`] instance always corresponds to a valid C `struct cpufreq_policy_data`. +/// +/// The callers must ensure that the `struct cpufreq_policy_data` is valid for access and remains +/// valid for the lifetime of the returned reference. +#[repr(transparent)] +pub struct PolicyData(Opaque); + +impl PolicyData { + /// Creates a mutable reference to an existing `struct cpufreq_policy_data` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy_data) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw pointer to the underlying C `cpufreq_policy_data`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_policy_data { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Wrapper for `cpufreq_generic_frequency_table_verify`. + #[inline] + pub fn generic_verify(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_frequency_table_verify(self.as_raw()) }) + } +} + +/// The frequency table index. +/// +/// Represents index with a frequency table. +/// +/// # Invariants +/// +/// The index must correspond to a valid entry in the [`Table`] it is used for. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct TableIndex(usize); + +impl TableIndex { + /// Creates an instance of [`TableIndex`]. + /// + /// # Safety + /// + /// The caller must ensure that `index` correspond to a valid entry in the [`Table`] it is used + /// for. + pub unsafe fn new(index: usize) -> Self { + // INVARIANT: The caller ensures that `index` correspond to a valid entry in the [`Table`]. + Self(index) + } +} + +impl From for usize { + #[inline] + fn from(index: TableIndex) -> Self { + index.0 + } +} + +/// CPU frequency table. +/// +/// Rust abstraction for the C `struct cpufreq_frequency_table`. +/// +/// # Invariants +/// +/// A [`Table`] instance always corresponds to a valid C `struct cpufreq_frequency_table`. +/// +/// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to read a frequency value from [`Table`]. +/// +/// ``` +/// use kernel::cpufreq::{Policy, TableIndex}; +/// +/// fn show_freq(policy: &Policy) -> Result { +/// let table = policy.freq_table()?; +/// +/// // SAFETY: Index is a valid entry in the table. +/// let index = unsafe { TableIndex::new(0) }; +/// +/// pr_info!("The frequency at index 0 is: {:?}\n", table.freq(index)?); +/// pr_info!("The flags at index 0 is: {}\n", table.flags(index)); +/// pr_info!("The data at index 0 is: {}\n", table.data(index)); +/// Ok(()) +/// } +/// ``` +#[repr(transparent)] +pub struct Table(Opaque); + +impl Table { + /// Creates a reference to an existing C `struct cpufreq_frequency_table` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_frequency_table) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Returns the raw mutable pointer to the C `struct cpufreq_frequency_table`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_frequency_table { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Returns frequency at `index` in the [`Table`]. + #[inline] + pub fn freq(&self, index: TableIndex) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + Ok(Hertz::from_khz(unsafe { + (*self.as_raw().add(index.into())).frequency.try_into()? + })) + } + + /// Returns flags at `index` in the [`Table`]. + #[inline] + pub fn flags(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).flags } + } + + /// Returns data at `index` in the [`Table`]. + #[inline] + pub fn data(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).driver_data } + } +} + +/// CPU frequency table owned and pinned in memory, created from a [`TableBuilder`]. +pub struct TableBox { + entries: Pin>, +} + +impl TableBox { + /// Constructs a new [`TableBox`] from a [`KVec`] of entries. + /// + /// # Errors + /// + /// Returns `EINVAL` if the entries list is empty. + #[inline] + fn new(entries: KVec) -> Result { + if entries.is_empty() { + return Err(EINVAL); + } + + Ok(Self { + // Pin the entries to memory, since we are passing its pointer to the C code. + entries: Pin::new(entries), + }) + } + + /// Returns a raw pointer to the underlying C `cpufreq_frequency_table`. + #[inline] + fn as_raw(&self) -> *const bindings::cpufreq_frequency_table { + // The pointer is valid until the table gets dropped. + self.entries.as_ptr() + } +} + +impl Deref for TableBox { + type Target = Table; + + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns TableBox, it is safe to deref. + unsafe { Self::Target::from_raw(self.as_raw()) } + } +} + +/// CPU frequency table builder. +/// +/// This is used by the CPU frequency drivers to build a frequency table dynamically. +/// +/// # Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{TableBuilder, TableIndex}; +/// use kernel::clk::Hertz; +/// +/// let mut builder = TableBuilder::new(); +/// +/// // Adds few entries to the table. +/// builder.add(Hertz::from_mhz(700), 0, 1).unwrap(); +/// builder.add(Hertz::from_mhz(800), 2, 3).unwrap(); +/// builder.add(Hertz::from_mhz(900), 4, 5).unwrap(); +/// builder.add(Hertz::from_ghz(1), 6, 7).unwrap(); +/// +/// let table = builder.to_table().unwrap(); +/// +/// // SAFETY: Index values correspond to valid entries in the table. +/// let (index0, index2) = unsafe { (TableIndex::new(0), TableIndex::new(2)) }; +/// +/// assert_eq!(table.freq(index0), Ok(Hertz::from_mhz(700))); +/// assert_eq!(table.flags(index0), 0); +/// assert_eq!(table.data(index0), 1); +/// +/// assert_eq!(table.freq(index2), Ok(Hertz::from_mhz(900))); +/// assert_eq!(table.flags(index2), 4); +/// assert_eq!(table.data(index2), 5); +/// ``` +#[derive(Default)] +#[repr(transparent)] +pub struct TableBuilder { + entries: KVec, +} + +impl TableBuilder { + /// Creates a new instance of [`TableBuilder`]. + #[inline] + pub fn new() -> Self { + Self { + entries: KVec::new(), + } + } + + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { + // Adds the new entry at the end of the vector. + Ok(self.entries.push( + bindings::cpufreq_frequency_table { + flags, + driver_data, + frequency, + }, + GFP_KERNEL, + )?) + } + + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. + pub fn to_table(mut self) -> Result { + // Add last entry to the table. + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; + + TableBox::new(self.entries) + } +} + +/// CPU frequency policy. +/// +/// Rust abstraction for the C `struct cpufreq_policy`. +/// +/// # Invariants +/// +/// A [`Policy`] instance always corresponds to a valid C `struct cpufreq_policy`. +/// +/// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid +/// for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{DEFAULT_TRANSITION_LATENCY_NS, Policy}; +/// +/// #[allow(clippy::double_parens, reason = "False positive before 1.92.0")] +/// fn update_policy(policy: &mut Policy) { +/// policy +/// .set_dvfs_possible_from_any_cpu(true) +/// .set_fast_switch_possible(true) +/// .set_transition_latency_ns(DEFAULT_TRANSITION_LATENCY_NS); +/// +/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur())); +/// } +/// ``` +#[repr(transparent)] +pub struct Policy(Opaque); + +impl Policy { + /// Creates a reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_policy) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Creates a mutable reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw mutable pointer to the C `struct cpufreq_policy`. + #[inline] + fn as_raw(&self) -> *mut bindings::cpufreq_policy { + let this: *const Self = self; + this.cast_mut().cast() + } + + #[inline] + fn as_ref(&self) -> &bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &*self.as_raw() } + } + + #[inline] + fn as_mut_ref(&mut self) -> &mut bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &mut *self.as_raw() } + } + + /// Returns the primary CPU for the [`Policy`]. + #[inline] + pub fn cpu(&self) -> CpuId { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + unsafe { CpuId::from_u32_unchecked(self.as_ref().cpu) } + } + + /// Returns the minimum frequency for the [`Policy`]. + #[inline] + pub fn min(&self) -> Hertz { + Hertz::from_khz(self.as_ref().min as usize) + } + + /// Set the minimum frequency for the [`Policy`]. + #[inline] + pub fn set_min(&mut self, min: Hertz) -> &mut Self { + self.as_mut_ref().min = min.as_khz() as u32; + self + } + + /// Returns the maximum frequency for the [`Policy`]. + #[inline] + pub fn max(&self) -> Hertz { + Hertz::from_khz(self.as_ref().max as usize) + } + + /// Set the maximum frequency for the [`Policy`]. + #[inline] + pub fn set_max(&mut self, max: Hertz) -> &mut Self { + self.as_mut_ref().max = max.as_khz() as u32; + self + } + + /// Returns the current frequency for the [`Policy`]. + #[inline] + pub fn cur(&self) -> Hertz { + Hertz::from_khz(self.as_ref().cur as usize) + } + + /// Returns the suspend frequency for the [`Policy`]. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + Hertz::from_khz(self.as_ref().suspend_freq as usize) + } + + /// Sets the suspend frequency for the [`Policy`]. + #[inline] + pub fn set_suspend_freq(&mut self, freq: Hertz) -> &mut Self { + self.as_mut_ref().suspend_freq = freq.as_khz() as u32; + self + } + + /// Provides a wrapper to the generic suspend routine. + #[inline] + pub fn generic_suspend(&mut self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_suspend(self.as_mut_ref()) }) + } + + /// Provides a wrapper to the generic get routine. + #[inline] + pub fn generic_get(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + Ok(unsafe { bindings::cpufreq_generic_get(u32::from(self.cpu())) }) + } + + /// Provides a wrapper to the register with energy model using the OPP core. + #[cfg(CONFIG_PM_OPP)] + #[inline] + pub fn register_em_opp(&mut self) { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { bindings::cpufreq_register_em_with_opp(self.as_mut_ref()) }; + } + + /// Gets [`cpumask::Cpumask`] for a cpufreq [`Policy`]. + #[inline] + pub fn cpus(&mut self) -> &mut cpumask::Cpumask { + // SAFETY: The pointer to `cpus` is valid for writing and remains valid for the lifetime of + // the returned reference. + unsafe { cpumask::CpumaskVar::from_raw_mut(&mut self.as_mut_ref().cpus) } + } + + /// Sets clock for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the returned [`Clk`] is not dropped while it is getting used + /// by the C code. + #[cfg(CONFIG_COMMON_CLK)] + pub unsafe fn set_clk(&mut self, dev: &Device, name: Option<&CStr>) -> Result { + let clk = Clk::get(dev, name)?; + self.as_mut_ref().clk = clk.as_raw(); + Ok(clk) + } + + /// Allows / disallows frequency switching code to run on any CPU. + #[inline] + pub fn set_dvfs_possible_from_any_cpu(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().dvfs_possible_from_any_cpu = val; + self + } + + /// Returns if fast switching of frequencies is possible or not. + #[inline] + pub fn fast_switch_possible(&self) -> bool { + self.as_ref().fast_switch_possible + } + + /// Enables / disables fast frequency switching. + #[inline] + pub fn set_fast_switch_possible(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().fast_switch_possible = val; + self + } + + /// Sets transition latency (in nanoseconds) for the [`Policy`]. + #[inline] + pub fn set_transition_latency_ns(&mut self, latency_ns: u32) -> &mut Self { + self.as_mut_ref().cpuinfo.transition_latency = latency_ns; + self + } + + /// Sets cpuinfo `min_freq`. + #[inline] + pub fn set_cpuinfo_min_freq(&mut self, min_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.min_freq = min_freq.as_khz() as u32; + self + } + + /// Sets cpuinfo `max_freq`. + #[inline] + pub fn set_cpuinfo_max_freq(&mut self, max_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.max_freq = max_freq.as_khz() as u32; + self + } + + /// Set `transition_delay_us`, i.e. the minimum time between successive frequency change + /// requests. + #[inline] + pub fn set_transition_delay_us(&mut self, transition_delay_us: u32) -> &mut Self { + self.as_mut_ref().transition_delay_us = transition_delay_us; + self + } + + /// Returns reference to the CPU frequency [`Table`] for the [`Policy`]. + pub fn freq_table(&self) -> Result<&Table> { + if self.as_ref().freq_table.is_null() { + return Err(EINVAL); + } + + // SAFETY: The `freq_table` is guaranteed to be valid for reading and remains valid for the + // lifetime of the returned reference. + Ok(unsafe { Table::from_raw(self.as_ref().freq_table) }) + } + + /// Sets the CPU frequency [`Table`] for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the [`Table`] is not dropped while it is getting used by the + /// C code. + #[inline] + pub unsafe fn set_freq_table(&mut self, table: &Table) -> &mut Self { + self.as_mut_ref().freq_table = table.as_raw(); + self + } + + /// Returns the [`Policy`]'s private data. + pub fn data(&mut self) -> Option<::Borrowed<'_>> { + if self.as_ref().driver_data.is_null() { + None + } else { + // SAFETY: The data is earlier set from [`set_data`]. + Some(unsafe { T::borrow(self.as_ref().driver_data.cast()) }) + } + } + + /// Sets the private data of the [`Policy`] using a foreign-ownable wrapper. + /// + /// # Errors + /// + /// Returns `EBUSY` if private data is already set. + fn set_data(&mut self, data: T) -> Result { + if self.as_ref().driver_data.is_null() { + // Transfer the ownership of the data to the foreign interface. + self.as_mut_ref().driver_data = ::into_foreign(data).cast(); + Ok(()) + } else { + Err(EBUSY) + } + } + + /// Clears and returns ownership of the private data. + fn clear_data(&mut self) -> Option { + if self.as_ref().driver_data.is_null() { + None + } else { + let data = Some( + // SAFETY: The data is earlier set by us from [`set_data`]. It is safe to take + // back the ownership of the data from the foreign interface. + unsafe { ::from_foreign(self.as_ref().driver_data.cast()) }, + ); + self.as_mut_ref().driver_data = ptr::null_mut(); + data + } + } +} + +/// CPU frequency policy created from a CPU number. +/// +/// This struct represents the CPU frequency policy obtained for a specific CPU, providing safe +/// access to the underlying `cpufreq_policy` and ensuring proper cleanup when the `PolicyCpu` is +/// dropped. +struct PolicyCpu<'a>(&'a mut Policy); + +impl<'a> PolicyCpu<'a> { + fn from_cpu(cpu: CpuId) -> Result { + // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. + let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?; + + Ok(Self( + // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of + // the returned reference. + unsafe { Policy::from_raw_mut(ptr) }, + )) + } +} + +impl<'a> Deref for PolicyCpu<'a> { + type Target = Policy; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a> DerefMut for PolicyCpu<'a> { + fn deref_mut(&mut self) -> &mut Policy { + self.0 + } +} + +impl<'a> Drop for PolicyCpu<'a> { + fn drop(&mut self) { + // SAFETY: The underlying pointer is guaranteed to be valid for the lifetime of `self`. + unsafe { bindings::cpufreq_cpu_put(self.0.as_raw()) }; + } +} + +/// CPU frequency driver. +/// +/// Implement this trait to provide a CPU frequency driver and its callbacks. +/// +/// Reference: +#[vtable] +pub trait Driver { + /// Driver's name. + const NAME: &'static CStr; + + /// Driver's flags. + const FLAGS: u16; + + /// Boost support. + const BOOST_ENABLED: bool; + + /// Policy specific data. + /// + /// Require that `PData` implements `ForeignOwnable`. We guarantee to never move the underlying + /// wrapped data structure. + type PData: ForeignOwnable; + + /// Driver's `init` callback. + fn init(policy: &mut Policy) -> Result; + + /// Driver's `exit` callback. + fn exit(_policy: &mut Policy, _data: Option) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `online` callback. + fn online(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `offline` callback. + fn offline(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `suspend` callback. + fn suspend(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `resume` callback. + fn resume(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `ready` callback. + fn ready(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `verify` callback. + fn verify(data: &mut PolicyData) -> Result; + + /// Driver's `setpolicy` callback. + fn setpolicy(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target` callback. + fn target(_policy: &mut Policy, _target_freq: u32, _relation: Relation) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_index` callback. + fn target_index(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `fast_switch` callback. + fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `adjust_perf` callback. + fn adjust_perf( + _policy: &mut Policy, + _min_perf: usize, + _target_perf: usize, + _max_perf: usize, + _capacity: usize, + ) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get_intermediate` callback. + fn get_intermediate(_policy: &mut Policy, _index: TableIndex) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_intermediate` callback. + fn target_intermediate(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get` callback. + fn get(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `update_limits` callback. + fn update_limits(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `bios_limit` callback. + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `set_boost` callback. + fn set_boost(_policy: &mut Policy, _state: i32) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `register_em` callback. + fn register_em(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// CPU frequency driver Registration. +/// +/// # Examples +/// +/// The following example demonstrates how to register a cpufreq driver. +/// +/// ``` +/// use kernel::{ +/// cpufreq, +/// device::{Core, Device}, +/// macros::vtable, +/// of, platform, +/// sync::Arc, +/// }; +/// struct SampleDevice; +/// +/// #[derive(Default)] +/// struct SampleDriver; +/// +/// #[vtable] +/// impl cpufreq::Driver for SampleDriver { +/// const NAME: &'static CStr = c"cpufreq-sample"; +/// const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV; +/// const BOOST_ENABLED: bool = true; +/// +/// type PData = Arc; +/// +/// fn init(policy: &mut cpufreq::Policy) -> Result { +/// // Initialize here +/// Ok(Arc::new(SampleDevice, GFP_KERNEL)?) +/// } +/// +/// fn exit(_policy: &mut cpufreq::Policy, _data: Option) -> Result { +/// Ok(()) +/// } +/// +/// fn suspend(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_suspend() +/// } +/// +/// fn verify(data: &mut cpufreq::PolicyData) -> Result { +/// data.generic_verify() +/// } +/// +/// fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result { +/// // Update CPU frequency +/// Ok(()) +/// } +/// +/// fn get(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_get() +/// } +/// } +/// +/// impl platform::Driver for SampleDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = None; +/// +/// fn probe<'bound>( +/// pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit + 'bound { +/// cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; +/// Ok(Self {}) +/// } +/// } +/// ``` +#[repr(transparent)] +pub struct Registration(KBox>, PhantomData); + +/// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads +/// or CPUs, so it is safe to share it. +unsafe impl Sync for Registration {} + +#[allow(clippy::non_send_fields_in_send_ty)] +/// SAFETY: Registration with and unregistration from the cpufreq subsystem can happen from any +/// thread. +unsafe impl Send for Registration {} + +impl Registration { + const VTABLE: bindings::cpufreq_driver = bindings::cpufreq_driver { + name: Self::copy_name(T::NAME), + boost_enabled: T::BOOST_ENABLED, + flags: T::FLAGS, + + // Initialize mandatory callbacks. + init: Some(Self::init_callback), + verify: Some(Self::verify_callback), + + // Initialize optional callbacks based on the traits of `T`. + setpolicy: if T::HAS_SETPOLICY { + Some(Self::setpolicy_callback) + } else { + None + }, + target: if T::HAS_TARGET { + Some(Self::target_callback) + } else { + None + }, + target_index: if T::HAS_TARGET_INDEX { + Some(Self::target_index_callback) + } else { + None + }, + fast_switch: if T::HAS_FAST_SWITCH { + Some(Self::fast_switch_callback) + } else { + None + }, + adjust_perf: if T::HAS_ADJUST_PERF { + Some(Self::adjust_perf_callback) + } else { + None + }, + get_intermediate: if T::HAS_GET_INTERMEDIATE { + Some(Self::get_intermediate_callback) + } else { + None + }, + target_intermediate: if T::HAS_TARGET_INTERMEDIATE { + Some(Self::target_intermediate_callback) + } else { + None + }, + get: if T::HAS_GET { + Some(Self::get_callback) + } else { + None + }, + update_limits: if T::HAS_UPDATE_LIMITS { + Some(Self::update_limits_callback) + } else { + None + }, + bios_limit: if T::HAS_BIOS_LIMIT { + Some(Self::bios_limit_callback) + } else { + None + }, + online: if T::HAS_ONLINE { + Some(Self::online_callback) + } else { + None + }, + offline: if T::HAS_OFFLINE { + Some(Self::offline_callback) + } else { + None + }, + exit: if T::HAS_EXIT { + Some(Self::exit_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Self::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Self::resume_callback) + } else { + None + }, + ready: if T::HAS_READY { + Some(Self::ready_callback) + } else { + None + }, + set_boost: if T::HAS_SET_BOOST { + Some(Self::set_boost_callback) + } else { + None + }, + register_em: if T::HAS_REGISTER_EM { + Some(Self::register_em_callback) + } else { + None + }, + ..pin_init::zeroed() + }; + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { + let src = name.to_bytes_with_nul(); + let mut dst = [0; CPUFREQ_NAME_LEN]; + + build_assert!(src.len() <= CPUFREQ_NAME_LEN); + + let mut i = 0; + while i < src.len() { + dst[i] = src[i]; + i += 1; + } + + dst + } + + /// Registers a CPU frequency driver with the cpufreq core. + pub fn new() -> Result { + // We can't use `&Self::VTABLE` directly because the cpufreq core modifies some fields in + // the C `struct cpufreq_driver`, which requires a mutable reference. + let mut drv = KBox::new(UnsafeCell::new(Self::VTABLE), GFP_KERNEL)?; + + // SAFETY: `drv` is guaranteed to be valid for the lifetime of `Registration`. + to_result(unsafe { bindings::cpufreq_register_driver(drv.get_mut()) })?; + + Ok(Self(drv, PhantomData)) + } + + /// Same as [`Registration::new`], but does not return a [`Registration`] instance. + /// + /// Instead the [`Registration`] is owned by [`devres::register`] and will be dropped, once the + /// device is detached. + pub fn new_foreign_owned(dev: &Device) -> Result + where + T: 'static, + { + devres::register(dev, Self::new()?, GFP_KERNEL) + } +} + +/// CPU frequency driver callbacks. +impl Registration { + /// Driver's `init` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = T::init(policy)?; + policy.set_data(data)?; + Ok(0) + }) + } + + /// Driver's `exit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = policy.clear_data(); + let _ = T::exit(policy, data); + } + + /// Driver's `online` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::online(policy).map(|()| 0) + }) + } + + /// Driver's `offline` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::offline(policy).map(|()| 0) + }) + } + + /// Driver's `suspend` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::suspend(policy).map(|()| 0) + }) + } + + /// Driver's `resume` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::resume(policy).map(|()| 0) + }) + } + + /// Driver's `ready` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::ready(policy); + } + + /// Driver's `verify` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let data = unsafe { PolicyData::from_raw_mut(ptr) }; + T::verify(data).map(|()| 0) + }) + } + + /// Driver's `setpolicy` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::setpolicy(policy).map(|()| 0) + }) + } + + /// Driver's `target` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: c_uint, + relation: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::target(policy, target_freq, Relation::new(relation)?).map(|()| 0) + }) + } + + /// Driver's `target_index` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_index_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_index(policy, index).map(|()| 0) + }) + } + + /// Driver's `fast_switch` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn fast_switch_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: c_uint, + ) -> c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::fast_switch(policy, target_freq) + } + + /// Driver's `adjust_perf` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn adjust_perf_callback( + ptr: *mut bindings::cpufreq_policy, + min_perf: c_ulong, + target_perf: c_ulong, + max_perf: c_ulong, + capacity: c_ulong, + ) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity); + } + + /// Driver's `get_intermediate` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn get_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::get_intermediate(policy, index) + } + + /// Driver's `target_intermediate` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_intermediate(policy, index).map(|()| 0) + }) + } + + /// Driver's `get` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + unsafe extern "C" fn get_callback(cpu: c_uint) -> c_uint { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; + + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).unwrap_or(0)) + } + + /// Driver's `update_limit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::update_limits(policy); + } + + /// Driver's `bios_limit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn bios_limit_callback(cpu: c_int, limit: *mut c_uint) -> c_int { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_i32_unchecked(cpu) }; + + from_result(|| { + let mut policy = PolicyCpu::from_cpu(cpu_id)?; + let val = T::bios_limit(&mut policy)?; + // SAFETY: `limit` is guaranteed by the C code to be valid. + unsafe { + *limit = val; + } + Ok(0) + }) + } + + /// Driver's `set_boost` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn set_boost_callback( + ptr: *mut bindings::cpufreq_policy, + state: c_int, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::set_boost(policy, state).map(|()| 0) + }) + } + + /// Driver's `register_em` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::register_em(policy); + } +} + +impl Drop for Registration { + /// Unregisters with the cpufreq core. + fn drop(&mut self) { + // SAFETY: `self.0` is guaranteed to be valid for the lifetime of `Registration`. + unsafe { bindings::cpufreq_unregister_driver(self.0.get_mut()) }; + } +} diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs new file mode 100644 index 000000000..44bb36636 --- /dev/null +++ b/rust/kernel/cpumask.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU Mask abstractions. +//! +//! C header: [`include/linux/cpumask.h`](srctree/include/linux/cpumask.h) + +use crate::{ + alloc::{AllocError, Flags}, + cpu::CpuId, + prelude::*, + types::Opaque, +}; + +#[cfg(CONFIG_CPUMASK_OFFSTACK)] +use core::ptr::{self, NonNull}; + +use core::ops::{Deref, DerefMut}; + +/// A CPU Mask. +/// +/// Rust abstraction for the C `struct cpumask`. +/// +/// # Invariants +/// +/// A [`Cpumask`] instance always corresponds to a valid C `struct cpumask`. +/// +/// The callers must ensure that the `struct cpumask` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to update a [`Cpumask`]. +/// +/// ``` +/// use kernel::bindings; +/// use kernel::cpu::CpuId; +/// use kernel::cpumask::Cpumask; +/// +/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: CpuId, clear_cpu: CpuId) { +/// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the +/// // returned reference. +/// let mask = unsafe { Cpumask::from_raw_mut(ptr) }; +/// +/// mask.set(set_cpu); +/// mask.clear(clear_cpu); +/// } +/// ``` +#[repr(transparent)] +pub struct Cpumask(Opaque); + +impl Cpumask { + /// Creates a mutable reference from an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference from an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Obtain the raw `struct cpumask` pointer. + pub fn as_raw(&self) -> *mut bindings::cpumask { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Set `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `set()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_set_cpu()`. + #[inline] + pub fn set(&mut self, cpu: CpuId) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`. + unsafe { bindings::__cpumask_set_cpu(u32::from(cpu), self.as_raw()) }; + } + + /// Clear `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `clear()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_clear_cpu()`. + #[inline] + pub fn clear(&mut self, cpu: CpuId) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to + // `__cpumask_clear_cpu`. + unsafe { bindings::__cpumask_clear_cpu(i32::from(cpu), self.as_raw()) }; + } + + /// Test `cpu` in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_test_cpu` API. + #[inline] + pub fn test(&self, cpu: CpuId) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`. + unsafe { bindings::cpumask_test_cpu(i32::from(cpu), self.as_raw()) } + } + + /// Set all CPUs in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_setall` API. + #[inline] + pub fn setall(&mut self) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_setall`. + unsafe { bindings::cpumask_setall(self.as_raw()) }; + } + + /// Checks if cpumask is empty. + /// + /// Equivalent to the kernel's `cpumask_empty` API. + #[inline] + pub fn empty(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_empty`. + unsafe { bindings::cpumask_empty(self.as_raw()) } + } + + /// Checks if cpumask is full. + /// + /// Equivalent to the kernel's `cpumask_full` API. + #[inline] + pub fn full(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_full`. + unsafe { bindings::cpumask_full(self.as_raw()) } + } + + /// Get weight of the cpumask. + /// + /// Equivalent to the kernel's `cpumask_weight` API. + #[inline] + pub fn weight(&self) -> u32 { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_weight`. + unsafe { bindings::cpumask_weight(self.as_raw()) } + } + + /// Copy cpumask. + /// + /// Equivalent to the kernel's `cpumask_copy` API. + #[inline] + pub fn copy(&self, dstp: &mut Self) { + // SAFETY: By the type invariant, `Self::as_raw` is a valid argument to `cpumask_copy`. + unsafe { bindings::cpumask_copy(dstp.as_raw(), self.as_raw()) }; + } +} + +/// A CPU Mask pointer. +/// +/// Rust abstraction for the C `struct cpumask_var_t`. +/// +/// # Invariants +/// +/// A [`CpumaskVar`] instance always corresponds to a valid C `struct cpumask_var_t`. +/// +/// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid +/// for the lifetime of [`CpumaskVar`]. +/// +/// # Examples +/// +/// The following example demonstrates how to create and update a [`CpumaskVar`]. +/// +/// ``` +/// use kernel::cpu::CpuId; +/// use kernel::cpumask::CpumaskVar; +/// +/// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap(); +/// +/// assert!(mask.empty()); +/// let mut count = 0; +/// +/// let cpu2 = CpuId::from_u32(2); +/// if let Some(cpu) = cpu2 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// let cpu3 = CpuId::from_u32(3); +/// if let Some(cpu) = cpu3 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// assert_eq!(mask.weight(), count); +/// +/// let mask2 = CpumaskVar::try_clone(&mask).unwrap(); +/// +/// if let Some(cpu) = cpu2 { +/// assert!(mask2.test(cpu)); +/// } +/// +/// if let Some(cpu) = cpu3 { +/// assert!(mask2.test(cpu)); +/// } +/// assert_eq!(mask2.weight(), count); +/// ``` +#[repr(transparent)] +pub struct CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: NonNull, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask, +} + +impl CpumaskVar { + /// Creates a zero-initialized instance of the [`CpumaskVar`]. + pub fn new_zero(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::zalloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask(Opaque::zeroed()), + }) + } + + /// Creates an instance of the [`CpumaskVar`]. + /// + /// # Safety + /// + /// The caller must ensure that the returned [`CpumaskVar`] is properly initialized before + /// getting used. + pub unsafe fn new(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::alloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask(Opaque::uninit()), + }) + } + + /// Creates a mutable reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask_var_t) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask_var_t) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Clones cpumask. + pub fn try_clone(cpumask: &Cpumask) -> Result { + // SAFETY: The returned cpumask_var is initialized right after this call. + let mut cpumask_var = unsafe { Self::new(GFP_KERNEL) }?; + + cpumask.copy(&mut cpumask_var); + Ok(cpumask_var) + } +} + +// Make [`CpumaskVar`] behave like a pointer to [`Cpumask`]. +impl Deref for CpumaskVar { + type Target = Cpumask; + + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { &*self.ptr.as_ptr() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref(&self) -> &Self::Target { + &self.mask + } +} + +impl DerefMut for CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref_mut(&mut self) -> &mut Cpumask { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { self.ptr.as_mut() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref_mut(&mut self) -> &mut Cpumask { + &mut self.mask + } +} + +impl Drop for CpumaskVar { + fn drop(&mut self) { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `free_cpumask_var`. + unsafe { + bindings::free_cpumask_var(self.as_raw()) + }; + } +} diff --git a/rust/kernel/cred.rs b/rust/kernel/cred.rs new file mode 100644 index 000000000..ffa156b9d --- /dev/null +++ b/rust/kernel/cred.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Credentials management. +//! +//! C header: [`include/linux/cred.h`](srctree/include/linux/cred.h). +//! +//! Reference: + +use crate::{bindings, sync::aref::AlwaysRefCounted, task::Kuid, types::Opaque}; + +/// Wraps the kernel's `struct cred`. +/// +/// Credentials are used for various security checks in the kernel. +/// +/// Most fields of credentials are immutable. When things have their credentials changed, that +/// happens by replacing the credential instead of changing an existing credential. See the [kernel +/// documentation][ref] for more info on this. +/// +/// # Invariants +/// +/// Instances of this type are always ref-counted, that is, a call to `get_cred` ensures that the +/// allocation remains valid at least until the matching call to `put_cred`. +/// +/// [ref]: https://www.kernel.org/doc/html/latest/security/credentials.html +#[repr(transparent)] +pub struct Credential(Opaque); + +// SAFETY: +// - `Credential::dec_ref` can be called from any thread. +// - It is okay to send ownership of `Credential` across thread boundaries. +unsafe impl Send for Credential {} + +// SAFETY: It's OK to access `Credential` through shared references from other threads because +// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for Credential {} + +impl Credential { + /// Creates a reference to a [`Credential`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Credential`] reference. + #[inline] + pub unsafe fn from_ptr<'a>(ptr: *const bindings::cred) -> &'a Credential { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Credential` type being transparent makes the cast ok. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to the inner credential. + #[inline] + pub fn as_ptr(&self) -> *const bindings::cred { + self.0.get() + } + + /// Get the id for this security context. + #[inline] + pub fn get_secid(&self) -> u32 { + let mut secid = 0; + // SAFETY: The invariants of this type ensures that the pointer is valid. + unsafe { bindings::security_cred_getsecid(self.0.get(), &mut secid) }; + secid + } + + /// Returns the effective UID of the given credential. + #[inline] + pub fn euid(&self) -> Kuid { + // SAFETY: By the type invariant, we know that `self.0` is valid. Furthermore, the `euid` + // field of a credential is never changed after initialization, so there is no potential + // for data races. + Kuid::from_raw(unsafe { (*self.0.get()).euid }) + } +} + +// SAFETY: The type invariants guarantee that `Credential` is always ref-counted. +unsafe impl AlwaysRefCounted for Credential { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_cred(self.0.get()) }; + } + + #[inline] + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. The cast is okay + // because `Credential` has the same representation as `struct cred`. + unsafe { bindings::put_cred(obj.cast().as_ptr()) }; + } +} diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs new file mode 100644 index 000000000..d7b8014a6 --- /dev/null +++ b/rust/kernel/debugfs.rs @@ -0,0 +1,724 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! DebugFS Abstraction +//! +//! C header: [`include/linux/debugfs.h`](srctree/include/linux/debugfs.h) + +// When DebugFS is disabled, many parameters are dead. Linting for this isn't helpful. +#![cfg_attr(not(CONFIG_DEBUG_FS), allow(unused_variables))] + +#[cfg(CONFIG_DEBUG_FS)] +use crate::sync::Arc; +use crate::{ + fmt, + prelude::*, + str::CStr, + uaccess::UserSliceReader, // +}; + +#[cfg(CONFIG_DEBUG_FS)] +use core::mem::ManuallyDrop; +use core::{ + marker::{ + PhantomData, + PhantomPinned, // + }, + ops::Deref, +}; + +mod traits; +pub use traits::{ + BinaryReader, + BinaryReaderMut, + BinaryWriter, + Reader, + Writer, // +}; + +mod callback_adapters; +use callback_adapters::{ + FormatAdapter, + NoWriter, + WritableAdapter, // +}; + +mod file_ops; +use file_ops::{ + BinaryReadFile, + BinaryReadWriteFile, + BinaryWriteFile, + FileOps, + ReadFile, + ReadWriteFile, + WriteFile, // +}; + +#[cfg(CONFIG_DEBUG_FS)] +mod entry; +#[cfg(CONFIG_DEBUG_FS)] +use entry::Entry; + +/// Owning handle to a DebugFS directory. +/// +/// The directory in the filesystem represented by [`Dir`] will be removed when handle has been +/// dropped *and* all children have been removed. +// If we have a parent, we hold a reference to it in the `Entry`. This prevents the `dentry` +// we point to from being cleaned up if our parent `Dir`/`Entry` is dropped before us. +// +// The `None` option indicates that the `Arc` could not be allocated, so our children would not be +// able to refer to us. In this case, we need to silently fail. All future child directories/files +// will silently fail as well. +#[derive(Clone)] +pub struct Dir(#[cfg(CONFIG_DEBUG_FS)] Option>>); + +impl Dir { + /// Create a new directory in DebugFS. If `parent` is [`None`], it will be created at the root. + fn create(name: &CStr, parent: Option<&Dir>) -> Self { + #[cfg(CONFIG_DEBUG_FS)] + { + let parent_entry = match parent { + // If the parent couldn't be allocated, just early-return + Some(Dir(None)) => return Self(None), + Some(Dir(Some(entry))) => Some(entry.clone()), + None => None, + }; + Self( + // If Arc creation fails, the `Entry` will be dropped, so the directory will be + // cleaned up. + Arc::new(Entry::dynamic_dir(name, parent_entry), GFP_KERNEL).ok(), + ) + } + #[cfg(not(CONFIG_DEBUG_FS))] + Self() + } + + /// Creates a DebugFS file which will own the data produced by the initializer provided in + /// `data`. + fn create_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + file_ops: &'static FileOps, + ) -> impl PinInit, E> + 'a + where + T: Sync + 'static, + { + let scope = Scope::::new(data, move |data| { + #[cfg(CONFIG_DEBUG_FS)] + if let Some(parent) = &self.0 { + // SAFETY: Because data derives from a scope, and our entry will be dropped before + // the data is dropped, it is guaranteed to outlive the entry we return. + unsafe { Entry::dynamic_file(name, parent.clone(), data, file_ops) } + } else { + Entry::empty() + } + }); + try_pin_init! { + File { + scope <- scope + } ? E + } + } + + /// Create a new directory in DebugFS at the root. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// let debugfs = Dir::new(c"parent"); + /// ``` + pub fn new(name: &CStr) -> Self { + Dir::create(name, None) + } + + /// Creates a subdirectory within this directory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// let parent = Dir::new(c"parent"); + /// let child = parent.subdir(c"child"); + /// ``` + pub fn subdir(&self, name: &CStr) -> Self { + Dir::create(name, Some(self)) + } + + /// Creates a read-only file in this directory. + /// + /// The file's contents are produced by invoking [`Writer::write`] on the value initialized by + /// `data`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// # use kernel::prelude::*; + /// # let dir = Dir::new(c"my_debugfs_dir"); + /// let file = KBox::pin_init(dir.read_only_file(c"foo", 200), GFP_KERNEL)?; + /// // "my_debugfs_dir/foo" now contains the number 200. + /// // The file is removed when `file` is dropped. + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_only_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Writer + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`] on the value + /// initialized by `data`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// # use kernel::prelude::*; + /// # let dir = Dir::new(c"my_debugfs_dir"); + /// let file = KBox::pin_init(dir.read_binary_file(c"foo", [0x1, 0x2]), GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryWriter + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only file in this directory, with contents from a callback. + /// + /// `f` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{ + /// # debugfs::Dir, + /// # prelude::*, + /// # sync::atomic::{ + /// # Atomic, + /// # Relaxed, + /// # }, + /// # }; + /// # let dir = Dir::new(c"foo"); + /// let file = KBox::pin_init( + /// dir.read_callback_file(c"bar", + /// Atomic::::new(3), + /// &|val, f| { + /// let out = val.load(Relaxed); + /// writeln!(f, "{out:#010x}") + /// }), + /// GFP_KERNEL)?; + /// // Reading "foo/bar" will show "0x00000003". + /// file.store(10, Relaxed); + /// // Reading "foo/bar" will now show "0x0000000a". + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_callback_file<'a, T, E: 'a, F>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _f: &'static F, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + { + let file_ops = >::FILE_OPS.adapt(); + self.create_file(name, data, file_ops) + } + + /// Creates a read-write file in this directory. + /// + /// Reading the file uses the [`Writer`] implementation. + /// Writing to the file uses the [`Reader`] implementation. + pub fn read_write_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Writer + Reader + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation. + /// Writing to the file uses the [`BinaryReader`] implementation. + pub fn read_write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryWriter + BinaryReader + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-write file in this directory, with logic from callbacks. + /// + /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. + /// + /// `f` and `w` must be function items or non-capturing closures. + /// This is statically asserted and not a safety requirement. + pub fn read_write_callback_file<'a, T, E: 'a, F, W>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _f: &'static F, + _w: &'static W, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let file_ops = + , W> as file_ops::ReadWriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, file_ops) + } + + /// Creates a write-only file in this directory. + /// + /// The file owns its backing data. Writing to the file uses the [`Reader`] + /// implementation. + /// + /// The file is removed when the returned [`File`] is dropped. + pub fn write_only_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Reader + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only binary file in this directory. + /// + /// The file owns its backing data. Writing to the file uses the [`BinaryReader`] + /// implementation. + /// + /// The file is removed when the returned [`File`] is dropped. + pub fn write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryReader + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only file in this directory, with write logic from a callback. + /// + /// `w` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + pub fn write_callback_file<'a, T, E: 'a, W>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _w: &'static W, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let file_ops = , W> as WriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, file_ops) + } + + // While this function is safe, it is intentionally not public because it's a bit of a + // footgun. + // + // Unless you also extract the `entry` later and schedule it for `Drop` at the appropriate + // time, a `ScopedDir` with a `Dir` parent will never be deleted. + fn scoped_dir<'data>(&self, name: &CStr) -> ScopedDir<'data, 'static> { + #[cfg(CONFIG_DEBUG_FS)] + { + let parent_entry = match &self.0 { + None => return ScopedDir::empty(), + Some(entry) => entry.clone(), + }; + ScopedDir { + entry: ManuallyDrop::new(Entry::dynamic_dir(name, Some(parent_entry))), + _phantom: PhantomData, + } + } + #[cfg(not(CONFIG_DEBUG_FS))] + ScopedDir::empty() + } + + /// Creates a new scope, which is a directory associated with some data `T`. + /// + /// The created directory will be a subdirectory of `self`. The `init` closure is called to + /// populate the directory with files and subdirectories. These files can reference the data + /// stored in the scope. + /// + /// The entire directory tree created within the scope will be removed when the returned + /// `Scope` handle is dropped. + pub fn scope<'a, T: 'a, E: 'a, F>( + &'a self, + data: impl PinInit + 'a, + name: &'a CStr, + init: F, + ) -> impl PinInit, E> + 'a + where + F: for<'data, 'dir> FnOnce(&'data T, &'dir ScopedDir<'data, 'dir>) + 'a, + { + Scope::new(data, |data| { + let scoped = self.scoped_dir(name); + init(data, &scoped); + scoped.into_entry() + }) + } +} + +#[pin_data] +/// Handle to a DebugFS scope, which ensures that attached `data` will outlive the DebugFS entry +/// without moving. +/// +/// This is internally used to back [`File`], and used in the API to represent the attachment +/// of a directory lifetime to a data structure which may be jointly accessed by a number of +/// different files. +/// +/// When dropped, a `Scope` will remove all directories and files in the filesystem backed by the +/// attached data structure prior to releasing the attached data. +pub struct Scope { + // This order is load-bearing for drops - `_entry` must be dropped before `data`. + #[cfg(CONFIG_DEBUG_FS)] + _entry: Entry<'static>, + #[pin] + data: T, + // Even if `T` is `Unpin`, we still can't allow it to be moved. + #[pin] + _pin: PhantomPinned, +} + +#[pin_data] +/// Handle to a DebugFS file, owning its backing data. +/// +/// When dropped, the DebugFS file will be removed and the attached data will be dropped. +pub struct File { + #[pin] + scope: Scope, +} + +#[cfg(not(CONFIG_DEBUG_FS))] +impl<'b, T: 'b> Scope { + fn new(data: impl PinInit + 'b, init: F) -> impl PinInit + 'b + where + F: for<'a> FnOnce(&'a T) + 'b, + { + try_pin_init! { + Self { + data <- data, + _pin: PhantomPinned + } ? E + } + .pin_chain(|scope| { + init(&scope.data); + Ok(()) + }) + } +} + +#[cfg(CONFIG_DEBUG_FS)] +impl<'b, T: 'b> Scope { + fn entry_mut(self: Pin<&mut Self>) -> &mut Entry<'static> { + // SAFETY: _entry is not structurally pinned. + unsafe { &mut Pin::into_inner_unchecked(self)._entry } + } + + fn new(data: impl PinInit + 'b, init: F) -> impl PinInit + 'b + where + F: for<'a> FnOnce(&'a T) -> Entry<'static> + 'b, + { + try_pin_init! { + Self { + _entry: Entry::empty(), + data <- data, + _pin: PhantomPinned + } ? E + } + .pin_chain(|scope| { + *scope.entry_mut() = init(&scope.data); + Ok(()) + }) + } +} + +impl<'a, T: 'a> Scope { + /// Creates a new scope, which is a directory at the root of the debugfs filesystem, + /// associated with some data `T`. + /// + /// The `init` closure is called to populate the directory with files and subdirectories. These + /// files can reference the data stored in the scope. + /// + /// The entire directory tree created within the scope will be removed when the returned + /// `Scope` handle is dropped. + pub fn dir( + data: impl PinInit + 'a, + name: &'a CStr, + init: F, + ) -> impl PinInit + 'a + where + F: for<'data, 'dir> FnOnce(&'data T, &'dir ScopedDir<'data, 'dir>) + 'a, + { + Scope::new(data, |data| { + let scoped = ScopedDir::new(name); + init(data, &scoped); + scoped.into_entry() + }) + } +} + +impl Deref for Scope { + type Target = T; + fn deref(&self) -> &T { + &self.data + } +} + +impl Deref for File { + type Target = T; + fn deref(&self) -> &T { + &self.scope + } +} + +/// A handle to a directory which will live at most `'dir`, accessing data that will live for at +/// least `'data`. +/// +/// Dropping a ScopedDir will not delete or clean it up, this is expected to occur through dropping +/// the `Scope` that created it. +pub struct ScopedDir<'data, 'dir> { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop>, + _phantom: PhantomData &'dir ()>, +} + +impl<'data, 'dir> ScopedDir<'data, 'dir> { + /// Creates a subdirectory inside this `ScopedDir`. + /// + /// The returned directory handle cannot outlive this one. + pub fn dir<'dir2>(&'dir2 self, name: &CStr) -> ScopedDir<'data, 'dir2> { + #[cfg(not(CONFIG_DEBUG_FS))] + let _ = name; + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::dir(name, Some(&*self.entry))), + _phantom: PhantomData, + } + } + + fn create_file(&self, name: &CStr, data: &'data T, vtable: &'static FileOps) { + #[cfg(CONFIG_DEBUG_FS)] + core::mem::forget(Entry::file(name, &self.entry, data, vtable)); + } + + /// Creates a read-only file in this directory. + /// + /// The file's contents are produced by invoking [`Writer::write`]. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_only_file(&self, name: &CStr, data: &'data T) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`]. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only file in this directory, with contents from a callback. + /// + /// The file contents are generated by calling `f` with `data`. + /// + /// + /// `f` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_callback_file(&self, name: &CStr, data: &'data T, _f: &'static F) + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + { + let vtable = as ReadFile<_>>::FILE_OPS.adapt(); + self.create_file(name, data, vtable) + } + + /// Creates a read-write file in this directory. + /// + /// Reading the file uses the [`Writer`] implementation on `data`. Writing to the file uses + /// the [`Reader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_write_file( + &self, + name: &CStr, + data: &'data T, + ) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation on `data`. Writing to the file + /// uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_write_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a read-write file in this directory, with logic from callbacks. + /// + /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. + /// + /// `f` and `w` must be function items or non-capturing closures. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_write_callback_file( + &self, + name: &CStr, + data: &'data T, + _f: &'static F, + _w: &'static W, + ) where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let vtable = , W> as ReadWriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, vtable) + } + + /// Creates a write-only file in this directory. + /// + /// Writing to the file uses the [`Reader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn write_only_file(&self, name: &CStr, data: &'data T) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a write-only binary file in this directory. + /// + /// Writing to the file uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn write_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only file in this directory, with write logic from a callback. + /// + /// Writing to the file is handled by `w`. + /// + /// `w` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn write_only_callback_file(&self, name: &CStr, data: &'data T, _w: &'static W) + where + T: Send + Sync + 'static, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let vtable = &, W> as WriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, vtable) + } + + fn empty() -> Self { + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::empty()), + _phantom: PhantomData, + } + } + #[cfg(CONFIG_DEBUG_FS)] + fn into_entry(self) -> Entry<'dir> { + ManuallyDrop::into_inner(self.entry) + } + #[cfg(not(CONFIG_DEBUG_FS))] + fn into_entry(self) {} +} + +impl<'data> ScopedDir<'data, 'static> { + // This is safe, but intentionally not exported due to footgun status. A ScopedDir with no + // parent will never be released by default, and needs to have its entry extracted and used + // somewhere. + fn new(name: &CStr) -> ScopedDir<'data, 'static> { + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::dir(name, None)), + _phantom: PhantomData, + } + } +} diff --git a/rust/kernel/debugfs/callback_adapters.rs b/rust/kernel/debugfs/callback_adapters.rs new file mode 100644 index 000000000..dee7d021e --- /dev/null +++ b/rust/kernel/debugfs/callback_adapters.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! Adapters which allow the user to supply a write or read implementation as a value rather +//! than a trait implementation. If provided, it will override the trait implementation. + +use super::{ + Reader, + Writer, // +}; + +use crate::{ + fmt, + prelude::*, + uaccess::UserSliceReader, // +}; + +use core::{ + marker::PhantomData, + ops::Deref, // +}; + +/// # Safety +/// +/// To implement this trait, it must be safe to cast a `&Self` to a `&Inner`. +/// It is intended for use in unstacking adapters out of `FileOps` backings. +pub(crate) unsafe trait Adapter { + type Inner; +} + +/// Adapter to implement `Reader` via a callback with the same representation as `T`. +/// +/// * Layer it on top of `WriterAdapter` if you want to add a custom callback for `write`. +/// * Layer it on top of `NoWriter` to pass through any support present on the underlying type. +/// +/// # Invariants +/// +/// If an instance for `WritableAdapter<_, W>` is constructed, `W` is inhabited. +#[repr(transparent)] +pub(crate) struct WritableAdapter { + inner: D, + _writer: PhantomData, +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for WritableAdapter { + type Inner = D; +} + +impl Writer for WritableAdapter { + fn write(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.write(fmt) + } +} + +impl Reader for WritableAdapter +where + W: Fn(&D::Target, &mut UserSliceReader) -> Result + Send + Sync + 'static, +{ + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + // SAFETY: WritableAdapter<_, W> can only be constructed if W is inhabited + let w: &W = unsafe { materialize_zst() }; + w(self.inner.deref(), reader) + } +} + +/// Adapter to implement `Writer` via a callback with the same representation as `T`. +/// +/// # Invariants +/// +/// If an instance for `FormatAdapter<_, F>` is constructed, `F` is inhabited. +#[repr(transparent)] +pub(crate) struct FormatAdapter { + inner: D, + _formatter: PhantomData, +} + +impl Deref for FormatAdapter { + type Target = D; + fn deref(&self) -> &D { + &self.inner + } +} + +impl Writer for FormatAdapter +where + F: Fn(&D, &mut fmt::Formatter<'_>) -> fmt::Result + 'static, +{ + fn write(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: FormatAdapter<_, F> can only be constructed if F is inhabited + let f: &F = unsafe { materialize_zst() }; + f(&self.inner, fmt) + } +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for FormatAdapter { + type Inner = D; +} + +#[repr(transparent)] +pub(crate) struct NoWriter { + inner: D, +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for NoWriter { + type Inner = D; +} + +impl Deref for NoWriter { + type Target = D; + fn deref(&self) -> &D { + &self.inner + } +} + +/// For types with a unique value, produce a static reference to it. +/// +/// # Safety +/// +/// The caller asserts that F is inhabited +unsafe fn materialize_zst() -> &'static F { + const { assert!(core::mem::size_of::() == 0) }; + let zst_dangle: core::ptr::NonNull = core::ptr::NonNull::dangling(); + // SAFETY: While the pointer is dangling, it is a dangling pointer to a ZST, based on the + // assertion above. The type is also inhabited, by the caller's assertion. This means + // we can materialize it. + unsafe { zst_dangle.as_ref() } +} diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs new file mode 100644 index 000000000..46aad6489 --- /dev/null +++ b/rust/kernel/debugfs/entry.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +use crate::{ + debugfs::file_ops::FileOps, + prelude::*, + str::{ + CStr, + CStrExt as _, // + }, + sync::Arc, +}; + +use core::marker::PhantomData; + +/// Owning handle to a DebugFS entry. +/// +/// # Invariants +/// +/// The wrapped pointer will always be `NULL`, an error, or an owned DebugFS `dentry`. +pub(crate) struct Entry<'a> { + entry: *mut bindings::dentry, + // If we were created with an owning parent, this is the keep-alive + _parent: Option>>, + // If we were created with a non-owning parent, this prevents us from outliving it + _phantom: PhantomData<&'a ()>, +} + +// SAFETY: [`Entry`] is just a `dentry` under the hood, which the API promises can be transferred +// between threads. +unsafe impl Send for Entry<'_> {} + +// SAFETY: All the C functions we call on the `dentry` pointer are threadsafe. +unsafe impl Sync for Entry<'_> {} + +impl Entry<'static> { + pub(crate) fn dynamic_dir(name: &CStr, parent: Option>) -> Self { + let parent_ptr = match &parent { + Some(entry) => entry.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent_ptr` is either `NULL` (if `parent` is `None`), or a pointer to a valid + // `dentry` by our invariant. `debugfs_create_dir` handles `NULL` pointers correctly. + let entry = unsafe { bindings::debugfs_create_dir(name.as_char_ptr(), parent_ptr) }; + + Entry { + entry, + _parent: parent, + _phantom: PhantomData, + } + } + + /// # Safety + /// + /// * `data` must outlive the returned `Entry`. + pub(crate) unsafe fn dynamic_file( + name: &CStr, + parent: Arc, + data: &T, + file_ops: &'static FileOps, + ) -> Self { + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent.as_ptr()` is a pointer to a valid `dentry` by invariant. + // * The caller guarantees that `data` will outlive the returned `Entry`. + // * The guarantees on `FileOps` assert the vtable will be compatible with the data we have + // provided. + let entry = unsafe { + bindings::debugfs_create_file_full( + name.as_char_ptr(), + file_ops.mode(), + parent.as_ptr(), + core::ptr::from_ref(data) as *mut c_void, + core::ptr::null(), + &**file_ops, + ) + }; + + Entry { + entry, + _parent: Some(parent), + _phantom: PhantomData, + } + } +} + +impl<'a> Entry<'a> { + pub(crate) fn dir(name: &CStr, parent: Option<&'a Entry<'_>>) -> Self { + let parent_ptr = match &parent { + Some(entry) => entry.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent_ptr` is either `NULL` (if `parent` is `None`), or a pointer to a valid + // `dentry` (because `parent` is a valid reference to an `Entry`). The lifetime `'a` + // ensures that the parent outlives this entry. + let entry = unsafe { bindings::debugfs_create_dir(name.as_char_ptr(), parent_ptr) }; + + Entry { + entry, + _parent: None, + _phantom: PhantomData, + } + } + + pub(crate) fn file( + name: &CStr, + parent: &'a Entry<'_>, + data: &'a T, + file_ops: &FileOps, + ) -> Self { + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent.as_ptr()` is a pointer to a valid `dentry` because we have `&'a Entry`. + // * `data` is a valid pointer to `T` for lifetime `'a`. + // * The returned `Entry` has lifetime `'a`, so it cannot outlive `parent` or `data`. + // * The caller guarantees that `vtable` is compatible with `data`. + // * The guarantees on `FileOps` assert the vtable will be compatible with the data we have + // provided. + let entry = unsafe { + bindings::debugfs_create_file_full( + name.as_char_ptr(), + file_ops.mode(), + parent.as_ptr(), + core::ptr::from_ref(data) as *mut c_void, + core::ptr::null(), + &**file_ops, + ) + }; + + Entry { + entry, + _parent: None, + _phantom: PhantomData, + } + } +} + +impl Entry<'_> { + /// Constructs a placeholder DebugFS [`Entry`]. + pub(crate) fn empty() -> Self { + Self { + entry: core::ptr::null_mut(), + _parent: None, + _phantom: PhantomData, + } + } + + /// Returns the pointer representation of the DebugFS directory. + /// + /// # Guarantees + /// + /// Due to the type invariant, the value returned from this function will always be an error + /// code, `NULL`, or a live DebugFS directory. If it is live, it will remain live at least as + /// long as this entry lives. + pub(crate) fn as_ptr(&self) -> *mut bindings::dentry { + self.entry + } +} + +impl Drop for Entry<'_> { + fn drop(&mut self) { + // SAFETY: `debugfs_remove` can take `NULL`, error values, and legal DebugFS dentries. + // `as_ptr` guarantees that the pointer is of this form. + unsafe { bindings::debugfs_remove(self.as_ptr()) } + } +} diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs new file mode 100644 index 000000000..f15908f71 --- /dev/null +++ b/rust/kernel/debugfs/file_ops.rs @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +use super::{ + BinaryReader, + BinaryWriter, + Reader, + Writer, // +}; + +use crate::{ + debugfs::callback_adapters::Adapter, + fmt, + fs::file, + prelude::*, + seq_file::SeqFile, + seq_print, + uaccess::UserSlice, // +}; + +use core::marker::PhantomData; + +#[cfg(CONFIG_DEBUG_FS)] +use core::ops::Deref; + +/// # Invariant +/// +/// `FileOps` will always contain an `operations` which is safe to use for a file backed +/// off an inode which has a pointer to a `T` in its private data that is safe to convert +/// into a reference. +pub(super) struct FileOps { + #[cfg(CONFIG_DEBUG_FS)] + operations: bindings::file_operations, + #[cfg(CONFIG_DEBUG_FS)] + mode: u16, + _phantom: PhantomData, +} + +impl FileOps { + /// # Safety + /// + /// The caller asserts that the provided `operations` is safe to use for a file whose + /// inode has a pointer to `T` in its private data that is safe to convert into a reference. + const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self { + Self { + #[cfg(CONFIG_DEBUG_FS)] + operations, + #[cfg(CONFIG_DEBUG_FS)] + mode, + _phantom: PhantomData, + } + } + + #[cfg(CONFIG_DEBUG_FS)] + pub(crate) const fn mode(&self) -> u16 { + self.mode + } +} + +impl FileOps { + pub(super) const fn adapt(&self) -> &FileOps { + // SAFETY: `Adapter` asserts that `T` can be legally cast to `T::Inner`. + unsafe { core::mem::transmute(self) } + } +} + +#[cfg(CONFIG_DEBUG_FS)] +impl Deref for FileOps { + type Target = bindings::file_operations; + + fn deref(&self) -> &Self::Target { + &self.operations + } +} + +struct WriterAdapter(T); + +impl<'a, T: Writer> fmt::Display for WriterAdapter<&'a T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.write(f) + } +} + +/// Implements `open` for `file_operations` via `single_open` to fill out a `seq_file`. +/// +/// # Safety +/// +/// * `inode`'s private pointer must point to a value of type `T` which will outlive the `inode` +/// and will not have any unique references alias it during the call. +/// * `file` must point to a live, not-yet-initialized file object. +unsafe extern "C" fn writer_open( + inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_int { + // SAFETY: The caller ensures that `inode` is a valid pointer. + let data = unsafe { (*inode).i_private }; + // SAFETY: + // * `file` is acceptable by caller precondition. + // * `print_act` will be called on a `seq_file` with private data set to the third argument, + // so we meet its safety requirements. + // * The `data` pointer passed in the third argument is a valid `T` pointer that outlives + // this call by caller preconditions. + unsafe { bindings::single_open(file, Some(writer_act::), data) } +} + +/// Prints private data stashed in a seq_file to that seq file. +/// +/// # Safety +/// +/// `seq` must point to a live `seq_file` whose private data is a valid pointer to a `T` which may +/// not have any unique references alias it during the call. +unsafe extern "C" fn writer_act( + seq: *mut bindings::seq_file, + _: *mut c_void, +) -> c_int { + // SAFETY: By caller precondition, this pointer is valid pointer to a `T`, and + // there are not and will not be any unique references until we are done. + let data = unsafe { &*((*seq).private.cast::()) }; + // SAFETY: By caller precondition, `seq_file` points to a live `seq_file`, so we can lift + // it. + let seq_file = unsafe { SeqFile::from_raw(seq) }; + seq_print!(seq_file, "{}", WriterAdapter(data)); + 0 +} + +// Work around lack of generic const items. +pub(crate) trait ReadFile { + const FILE_OPS: FileOps; +} + +impl ReadFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(bindings::seq_read), + llseek: Some(bindings::seq_lseek), + release: Some(bindings::single_release), + open: Some(writer_open::), + ..pin_init::zeroed() + }; + // SAFETY: `operations` is all stock `seq_file` implementations except for `writer_open`. + // `open`'s only requirement beyond what is provided to all open functions is that the + // inode's data pointer must point to a `T` that will outlive it, which matches the + // `FileOps` requirements. + unsafe { FileOps::new(operations, 0o400) } + }; +} + +fn read(data: &T, buf: *const c_char, count: usize) -> isize { + let mut reader = UserSlice::new(UserPtr::from_ptr(buf as *mut c_void), count).reader(); + + if let Err(e) = data.read_from_slice(&mut reader) { + return e.to_errno() as isize; + } + + count as isize +} + +/// # Safety +/// +/// `file` must be a valid pointer to a `file` struct. +/// The `private_data` of the file must contain a valid pointer to a `seq_file` whose +/// `private` data in turn points to a `T` that implements `Reader`. +/// `buf` must be a valid user-space buffer. +pub(crate) unsafe extern "C" fn write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + _ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: The file was opened with `single_open`, which sets `private_data` to a `seq_file`. + let seq = unsafe { &mut *((*file).private_data.cast::()) }; + // SAFETY: By caller precondition, this pointer is live and points to a value of type `T`. + let data = unsafe { &*(seq.private as *const T) }; + read(data, buf, count) +} + +// A trait to get the file operations for a type. +pub(crate) trait ReadWriteFile { + const FILE_OPS: FileOps; +} + +impl ReadWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + open: Some(writer_open::), + read: Some(bindings::seq_read), + write: Some(write::), + llseek: Some(bindings::seq_lseek), + release: Some(bindings::single_release), + ..pin_init::zeroed() + }; + // SAFETY: `operations` is all stock `seq_file` implementations except for `writer_open` + // and `write`. + // `writer_open`'s only requirement beyond what is provided to all open functions is that + // the inode's data pointer must point to a `T` that will outlive it, which matches the + // `FileOps` requirements. + // `write` only requires that the file's private data pointer points to `seq_file` + // which points to a `T` that will outlive it, which matches what `writer_open` + // provides. + unsafe { FileOps::new(operations, 0o600) } + }; +} + +/// # Safety +/// +/// `inode` must be a valid pointer to an `inode` struct. +/// `file` must be a valid pointer to a `file` struct. +unsafe extern "C" fn write_only_open( + inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_int { + // SAFETY: The caller ensures that `inode` and `file` are valid pointers. + unsafe { (*file).private_data = (*inode).i_private }; + 0 +} + +/// # Safety +/// +/// * `file` must be a valid pointer to a `file` struct. +/// * The `private_data` of the file must contain a valid pointer to a `T` that implements +/// `Reader`. +/// * `buf` must be a valid user-space buffer. +pub(crate) unsafe extern "C" fn write_only_write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + _ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: The caller ensures that `file` is a valid pointer and that `private_data` holds a + // valid pointer to `T`. + let data = unsafe { &*((*file).private_data as *const T) }; + read(data, buf, count) +} + +pub(crate) trait WriteFile { + const FILE_OPS: FileOps; +} + +impl WriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + open: Some(write_only_open), + write: Some(write_only_write::), + llseek: Some(bindings::noop_llseek), + ..pin_init::zeroed() + }; + // SAFETY: + // * `write_only_open` populates the file private data with the inode private data + // * `write_only_write`'s only requirement is that the private data of the file point to + // a `T` and be legal to convert to a shared reference, which `write_only_open` + // satisfies. + unsafe { FileOps::new(operations, 0o200) } + }; +} + +extern "C" fn blob_read( + file: *mut bindings::file, + buf: *mut c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut writer = UserSlice::new(UserPtr::from_ptr(buf.cast()), count).writer(); + + let ret = || -> Result { + let written = this.write_to_slice(&mut writer, pos)?; + + Ok(written.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for read only binary files. +pub(crate) trait BinaryReadFile { + const FILE_OPS: FileOps; +} + +impl BinaryReadFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(blob_read::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o400) } + }; +} + +extern "C" fn blob_write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut reader = UserSlice::new(UserPtr::from_ptr(buf.cast_mut().cast()), count).reader(); + + let ret = || -> Result { + let read = this.read_from_slice(&mut reader, pos)?; + + Ok(read.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for write only binary files. +pub(crate) trait BinaryWriteFile { + const FILE_OPS: FileOps; +} + +impl BinaryWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + write: Some(blob_write::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o200) } + }; +} + +/// Representation of [`FileOps`] for read/write binary files. +pub(crate) trait BinaryReadWriteFile { + const FILE_OPS: FileOps; +} + +impl BinaryReadWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(blob_read::), + write: Some(blob_write::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o600) } + }; +} diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs new file mode 100644 index 000000000..b295f8420 --- /dev/null +++ b/rust/kernel/debugfs/traits.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! Traits for rendering or updating values exported to DebugFS. + +use crate::{ + alloc::Allocator, + fmt, + fs::file, + prelude::*, + sync::{ + atomic::{ + Atomic, + AtomicBasicOps, + AtomicType, + Relaxed, // + }, + Arc, + Mutex, // + }, + uaccess::{ + UserSliceReader, + UserSliceWriter, // + }, +}; + +use core::{ + ops::{ + Deref, + DerefMut, // + }, + str::FromStr, +}; + +use zerocopy::Immutable; + +/// A trait for types that can be written into a string. +/// +/// This works very similarly to `Debug`, and is automatically implemented if `Debug` is +/// implemented for a type. It is also implemented for any writable type inside a `Mutex`. +/// +/// The derived implementation of `Debug` [may +/// change](https://doc.rust-lang.org/std/fmt/trait.Debug.html#stability) +/// between Rust versions, so if stability is key for your use case, please implement `Writer` +/// explicitly instead. +pub trait Writer { + /// Formats the value using the given formatter. + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result; +} + +impl Writer for Mutex { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.lock().write(f) + } +} + +impl Writer for T { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{self:?}") + } +} + +/// Trait for types that can be written out as binary. +pub trait BinaryWriter { + /// Writes the binary form of `self` into `writer`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes written in to `writer`. + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result; +} + +// Base implementation for any `T: Immutable + IntoBytes`. +impl BinaryWriter for T { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + writer.write_slice_file(self.as_bytes(), offset) + } +} + +// Delegate for `Mutex`: Support a `T` with an outer mutex. +impl BinaryWriter for Mutex { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + let guard = self.lock(); + + guard.write_to_slice(writer, offset) + } +} + +// Delegate for `Box`: Support a `Box` with no lock or an inner lock. +impl BinaryWriter for Box +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Pin>`: Support a `Pin>` with no lock or an inner lock. +impl BinaryWriter for Pin> +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Arc`: Support a `Arc` with no lock or an inner lock. +impl BinaryWriter for Arc +where + T: BinaryWriter, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Vec`. +impl BinaryWriter for Vec +where + T: Immutable + IntoBytes, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + writer.write_slice_file(self.as_bytes(), offset) + } +} + +/// A trait for types that can be updated from a user slice. +/// +/// This works similarly to `FromStr`, but operates on a `UserSliceReader` rather than a &str. +/// +/// It is automatically implemented for all atomic integers, or any type that implements `FromStr` +/// wrapped in a `Mutex`. +pub trait Reader { + /// Updates the value from the given user slice. + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result; +} + +impl Reader for Mutex { + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + let mut buf = [0u8; 128]; + if reader.len() > buf.len() { + return Err(EINVAL); + } + let n = reader.len(); + reader.read_slice(&mut buf[..n])?; + + let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; + let val = s.trim().parse::().map_err(|_| EINVAL)?; + *self.lock() = val; + Ok(()) + } +} + +impl Reader for Atomic +where + T::Repr: AtomicBasicOps, +{ + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + let mut buf = [0u8; 21]; // Enough for a 64-bit number. + if reader.len() > buf.len() { + return Err(EINVAL); + } + let n = reader.len(); + reader.read_slice(&mut buf[..n])?; + + let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; + let val = s.trim().parse::().map_err(|_| EINVAL)?; + self.store(val, Relaxed); + Ok(()) + } +} + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReader`] for interior mutability. +pub trait BinaryReaderMut { + /// Reads the binary form of `self` from `reader`. + /// + /// Same as [`BinaryReader::read_from_slice`], but takes a mutable reference. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result; +} + +// Base implementation for any `T: FromBytes + IntoBytes`. +impl BinaryReaderMut for T { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + reader.read_slice_file(self.as_mut_bytes(), offset) + } +} + +// Delegate for `Box`: Support a `Box` with an outer lock. +impl BinaryReaderMut for Box { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref_mut().read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Vec`: Support a `Vec` with an outer lock. +impl BinaryReaderMut for Vec +where + T: FromBytes + IntoBytes, + A: Allocator, +{ + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + reader.read_slice_file(self.as_mut_bytes(), offset) + } +} + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReaderMut`] for the mutable version. +pub trait BinaryReader { + /// Reads the binary form of `self` from `reader`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result; +} + +// Delegate for `Mutex`: Support a `T` with an outer `Mutex`. +impl BinaryReader for Mutex { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + let mut this = self.lock(); + + this.read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Box`: Support a `Box` with an inner lock. +impl BinaryReader for Box { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Pin>`: Support a `Pin>` with an inner lock. +impl BinaryReader for Pin> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Arc`: Support an `Arc` with an inner lock. +impl BinaryReader for Arc { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs new file mode 100644 index 000000000..2291d85b6 --- /dev/null +++ b/rust/kernel/device.rs @@ -0,0 +1,952 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic devices that are part of the kernel's driver model. +//! +//! C header: [`include/linux/device.h`](srctree/include/linux/device.h) + +use crate::{ + bindings, + fmt, + prelude::*, + sync::aref::ARef, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::{ + marker::PhantomData, + ptr, // +}; + +pub mod property; + +/// The core representation of a device in the kernel's driver model. +/// +/// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either +/// exist as temporary reference (see also [`Device::from_raw`]), which is only valid within a +/// certain scope or as [`ARef`], owning a dedicated reference count. +/// +/// # Device Types +/// +/// A [`Device`] can represent either a bus device or a class device. +/// +/// ## Bus Devices +/// +/// A bus device is a [`Device`] that is associated with a physical or virtual bus. Examples of +/// buses include PCI, USB, I2C, and SPI. Devices attached to a bus are registered with a specific +/// bus type, which facilitates matching devices with appropriate drivers based on IDs or other +/// identifying information. Bus devices are visible in sysfs under `/sys/bus//devices/`. +/// +/// ## Class Devices +/// +/// A class device is a [`Device`] that is associated with a logical category of functionality +/// rather than a physical bus. Examples of classes include block devices, network interfaces, sound +/// cards, and input devices. Class devices are grouped under a common class and exposed to +/// userspace via entries in `/sys/class//`. +/// +/// # Device Context +/// +/// [`Device`] references are generic over a [`DeviceContext`], which represents the type state of +/// a [`Device`]. +/// +/// As the name indicates, this type state represents the context of the scope the [`Device`] +/// reference is valid in. For instance, the [`Bound`] context guarantees that the [`Device`] is +/// bound to a driver for the entire duration of the existence of a [`Device`] reference. +/// +/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`], [`CoreInternal`] and +/// [`BoundInternal`]. +/// +/// Unless selected otherwise [`Device`] defaults to the [`Normal`] [`DeviceContext`], which by +/// itself has no additional requirements. +/// +/// It is always up to the caller of [`Device::from_raw`] to select the correct [`DeviceContext`] +/// type for the corresponding scope the [`Device`] reference is created in. +/// +/// All [`DeviceContext`] types other than [`Normal`] are intended to be used with +/// [bus devices](#bus-devices) only. +/// +/// # Implementing Bus Devices +/// +/// This section provides a guideline to implement bus specific devices, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`] +/// +/// A bus specific device should be defined as follows. +/// +/// ```ignore +/// #[repr(transparent)] +/// pub struct Device( +/// Opaque, +/// PhantomData, +/// ); +/// ``` +/// +/// Since devices are reference counted, [`AlwaysRefCounted`] should be implemented for `Device` +/// (i.e. `Device`). Note that [`AlwaysRefCounted`] must not be implemented for any other +/// [`DeviceContext`], since all other device context types are only valid within a certain scope. +/// +/// In order to be able to implement the [`DeviceContext`] dereference hierarchy, bus device +/// implementations should call the [`impl_device_context_deref`] macro as shown below. +/// +/// ```ignore +/// // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s +/// // generic argument. +/// kernel::impl_device_context_deref!(unsafe { Device }); +/// ``` +/// +/// In order to convert from a any [`Device`] to [`ARef`], bus devices can implement +/// the following macro call. +/// +/// ```ignore +/// kernel::impl_device_context_into_aref!(Device); +/// ``` +/// +/// Bus devices should also implement the following [`AsRef`] implementation, such that users can +/// easily derive a generic [`Device`] reference. +/// +/// ```ignore +/// impl AsRef> for Device { +/// fn as_ref(&self) -> &device::Device { +/// ... +/// } +/// } +/// ``` +/// +/// # Implementing Class Devices +/// +/// Class device implementations require less infrastructure and depend slightly more on the +/// specific subsystem. +/// +/// An example implementation for a class device could look like this. +/// +/// ```ignore +/// #[repr(C)] +/// pub struct Device { +/// dev: Opaque, +/// data: T::Data, +/// } +/// ``` +/// +/// This class device uses the sub-classing pattern to embed the driver's private data within the +/// allocation of the class device. For this to be possible the class device is generic over the +/// class specific `Driver` trait implementation. +/// +/// Just like any device, class devices are reference counted and should hence implement +/// [`AlwaysRefCounted`] for `Device`. +/// +/// Class devices should also implement the following [`AsRef`] implementation, such that users can +/// easily derive a generic [`Device`] reference. +/// +/// ```ignore +/// impl AsRef for Device { +/// fn as_ref(&self) -> &device::Device { +/// ... +/// } +/// } +/// ``` +/// +/// An example for a class device implementation is +#[cfg_attr(CONFIG_DRM = "y", doc = "[`drm::Device`](kernel::drm::Device).")] +#[cfg_attr(not(CONFIG_DRM = "y"), doc = "`drm::Device`.")] +/// +/// # Invariants +/// +/// A `Device` instance represents a valid `struct device` created by the C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to `get_device` ensures +/// that the allocation remains valid at least until the matching call to `put_device`. +/// +/// `bindings::device::release` is valid to be called from any thread, hence `ARef` can be +/// dropped from any thread. +/// +/// [`AlwaysRefCounted`]: kernel::sync::aref::AlwaysRefCounted +/// [`impl_device_context_deref`]: kernel::impl_device_context_deref +/// [`platform::Device`]: kernel::platform::Device +#[repr(transparent)] +pub struct Device(Opaque, PhantomData); + +impl Device { + /// Creates a new reference-counted abstraction instance of an existing `struct device` pointer. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to + /// can't drop to zero, for the duration of this function call. + /// + /// It must also be ensured that `bindings::device::release` can be called from any thread. + /// While not officially documented, this should be the case for any `struct device`. + pub unsafe fn get_device(ptr: *mut bindings::device) -> ARef { + // SAFETY: By the safety requirements ptr is valid + unsafe { Self::from_raw(ptr) }.into() + } + + /// Convert a [`&Device`](Device) into a [`&Device`](Device). + /// + /// # Safety + /// + /// The caller is responsible to ensure that the returned [`&Device`](Device) + /// only lives as long as it can be guaranteed that the [`Device`] is actually bound. + pub unsafe fn as_bound(&self) -> &Device { + let ptr = core::ptr::from_ref(self); + + // CAST: By the safety requirements the caller is responsible to guarantee that the + // returned reference only lives as long as the device is actually bound. + let ptr = ptr.cast(); + + // SAFETY: + // - `ptr` comes from `from_ref(self)` above, hence it's guaranteed to be valid. + // - Any valid `Device` pointer is also a valid pointer for `Device`. + unsafe { &*ptr } + } +} + +impl<'a> Device> { + /// Store a pointer to the bound driver's private data. + pub fn set_drvdata(&self, data: impl PinInit) -> Result { + let data = KBox::pin_init(data, GFP_KERNEL)?; + + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) }; + + Ok(()) + } + + /// Take ownership of the private data stored in this [`Device`]. + /// + /// # Safety + /// + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) }; + + if ptr.is_null() { + return None; + } + + // SAFETY: + // - If `ptr` is not NULL, it comes from a previous call to `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + Some(unsafe { Pin::>::from_foreign(ptr.cast()) }) + } +} + +impl Device { + /// Borrow the driver's private data bound to this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before the + /// device is fully unbound. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_borrow(&self) -> Pin<&T> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { Pin::>::borrow(ptr.cast()) } + } +} + +impl Device { + /// Obtain the raw `struct device *`. + pub(crate) fn as_raw(&self) -> *mut bindings::device { + self.0.get() + } + + /// Returns a reference to the parent device, if any. + #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] + pub(crate) fn parent(&self) -> Option<&Device> { + // SAFETY: + // - By the type invariant `self.as_raw()` is always valid. + // - The parent device is only ever set at device creation. + let parent = unsafe { (*self.as_raw()).parent }; + + if parent.is_null() { + None + } else { + // SAFETY: + // - Since `parent` is not NULL, it must be a valid pointer to a `struct device`. + // - `parent` is valid for the lifetime of `self`, since a `struct device` holds a + // reference count of its parent. + Some(unsafe { Device::from_raw(parent) }) + } + } + + /// Convert a raw C `struct device` pointer to a `&'a Device`. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to + /// can't drop to zero, for the duration of this function call and the entire duration when the + /// returned reference exists. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::device) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + unsafe { &*ptr.cast() } + } + + /// Prints an emergency-level message (level 0) prefixed with device information. + /// + /// More details are available from [`dev_emerg`]. + /// + /// [`dev_emerg`]: crate::dev_emerg + pub fn pr_emerg(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_EMERG, args) }; + } + + /// Prints an alert-level message (level 1) prefixed with device information. + /// + /// More details are available from [`dev_alert`]. + /// + /// [`dev_alert`]: crate::dev_alert + pub fn pr_alert(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ALERT, args) }; + } + + /// Prints a critical-level message (level 2) prefixed with device information. + /// + /// More details are available from [`dev_crit`]. + /// + /// [`dev_crit`]: crate::dev_crit + pub fn pr_crit(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_CRIT, args) }; + } + + /// Prints an error-level message (level 3) prefixed with device information. + /// + /// More details are available from [`dev_err`]. + /// + /// [`dev_err`]: crate::dev_err + pub fn pr_err(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ERR, args) }; + } + + /// Prints a warning-level message (level 4) prefixed with device information. + /// + /// More details are available from [`dev_warn`]. + /// + /// [`dev_warn`]: crate::dev_warn + pub fn pr_warn(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_WARNING, args) }; + } + + /// Prints a notice-level message (level 5) prefixed with device information. + /// + /// More details are available from [`dev_notice`]. + /// + /// [`dev_notice`]: crate::dev_notice + pub fn pr_notice(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_NOTICE, args) }; + } + + /// Prints an info-level message (level 6) prefixed with device information. + /// + /// More details are available from [`dev_info`]. + /// + /// [`dev_info`]: crate::dev_info + pub fn pr_info(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_INFO, args) }; + } + + /// Prints a debug-level message (level 7) prefixed with device information. + /// + /// More details are available from [`dev_dbg`]. + /// + /// [`dev_dbg`]: crate::dev_dbg + pub fn pr_dbg(&self, args: fmt::Arguments<'_>) { + if cfg!(debug_assertions) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_DEBUG, args) }; + } + } + + /// Prints the provided message to the console. + /// + /// # Safety + /// + /// Callers must ensure that `klevel` is null-terminated; in particular, one of the + /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. + #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] + unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated and one of the kernel constants. `self.as_raw` + // is valid because `self` is valid. The "%pA" format string expects a pointer to + // `fmt::Arguments`, which is what we're passing as the last argument. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_dev_printk( + klevel.as_ptr().cast::(), + self.as_raw(), + c"%pA".as_char_ptr(), + core::ptr::from_ref(&msg).cast::(), + ) + }; + } + + /// Obtain the [`FwNode`](property::FwNode) corresponding to this [`Device`]. + pub fn fwnode(&self) -> Option<&property::FwNode> { + // SAFETY: `self` is valid. + let fwnode_handle = unsafe { bindings::__dev_fwnode(self.as_raw()) }; + if fwnode_handle.is_null() { + return None; + } + // SAFETY: `fwnode_handle` is valid. Its lifetime is tied to `&self`. We + // return a reference instead of an `ARef` because `dev_fwnode()` + // doesn't increment the refcount. It is safe to cast from a + // `struct fwnode_handle*` to a `*const FwNode` because `FwNode` is + // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. + Some(unsafe { &*fwnode_handle.cast() }) + } + + /// Returns the name of the device. + /// + /// This is the kobject name of the device, or its initial name if the kobject is not yet + /// available. + #[inline] + pub fn name(&self) -> &CStr { + // SAFETY: By its type invariant `self.as_raw()` is a valid pointer to a `struct device`. + // The returned string is valid for the lifetime of the device. + unsafe { CStr::from_char_ptr(bindings::dev_name(self.as_raw())) } + } +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(obj.cast().as_ptr()) } + } +} + +// SAFETY: As by the type invariant `Device` can be sent to any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all immutable methods are protected by the +// synchronization in `struct device`. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct device` is the same; `Bound` is a +// zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Marker trait for the context or scope of a bus specific device. +/// +/// [`DeviceContext`] is a marker trait for types representing the context of a bus specific +/// [`Device`]. +/// +/// The specific device context types are: [`CoreInternal`], [`Core`], [`BoundInternal`], [`Bound`] +/// and [`Normal`]. +/// +/// [`DeviceContext`] types are hierarchical, which means that there is a strict hierarchy that +/// defines which [`DeviceContext`] type can be derived from another. For instance, any +/// [`Device`] can dereference to a [`Device`]. +/// +/// The following enumeration illustrates the dereference hierarchy of [`DeviceContext`] types. +/// +/// - [`CoreInternal`] => [`Core`] => [`Bound`] => [`Normal`] +/// - [`BoundInternal`] => [`Bound`] => [`Normal`] +/// +/// Both [`CoreInternal`] and [`BoundInternal`] implement the [`InternalBoundContext`] trait, +/// which provides access to internal bus abstraction methods on [`Device`] that are not available +/// to drivers. +/// +/// Bus devices can automatically implement the dereference hierarchy by using +/// [`impl_device_context_deref`]. +/// +/// Note that the guarantee for a [`Device`] reference to have a certain [`DeviceContext`] comes +/// from the specific scope the [`Device`] reference is valid in. +/// +/// [`impl_device_context_deref`]: kernel::impl_device_context_deref +pub trait DeviceContext: private::Sealed {} + +/// The [`Normal`] context is the default [`DeviceContext`] of any [`Device`]. +/// +/// The normal context does not indicate any specific context. Any `Device` is also a valid +/// [`Device`]. It is the only [`DeviceContext`] for which it is valid to implement +/// [`AlwaysRefCounted`] for. +/// +/// [`AlwaysRefCounted`]: kernel::sync::aref::AlwaysRefCounted +pub struct Normal; + +/// The [`Core`] context is the context of a bus specific device when it appears as argument of +/// any bus specific callback, such as `probe()`. +/// +/// The core context indicates that the [`Device`] reference's scope is limited to the bus +/// callback it appears in. It is intended to be used for synchronization purposes. Bus device +/// implementations can implement methods for [`Device`], such that they can only be called +/// from bus callbacks. +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData &'a ()>); + +/// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal core context is intended to be used in exactly the same way as the [`Core`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context mainly exists to share generic [`Device`] infrastructure that should only be called +/// from bus callbacks with bus abstractions, but without making them accessible for drivers. +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData &'a ()>); + +/// Semantically the same as [`Bound`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal bound context is intended to be used in exactly the same way as the [`Bound`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context exists for cases where the bus abstraction needs access to internal device +/// infrastructure (such as [`Device::drvdata_borrow`]), where [`CoreInternal`] would not be +/// justified. +pub struct BoundInternal; + +/// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to +/// be bound to a driver. +/// +/// The bound context indicates that for the entire duration of the lifetime of a [`Device`] +/// reference, the [`Device`] is guaranteed to be bound to a driver. +/// +/// Some APIs, such as [`dma::Coherent`] or [`Devres`] rely on the [`Device`] to be bound, +/// which can be proven with the [`Bound`] device context. +/// +/// Any abstraction that can guarantee a scope where the corresponding bus device is bound, should +/// provide a [`Device`] reference to its users for this scope. This allows users to benefit +/// from optimizations for accessing device resources, see also [`Devres::access`]. +/// +/// [`Devres`]: kernel::devres::Devres +/// [`Devres::access`]: kernel::devres::Devres::access +/// [`dma::Coherent`]: kernel::dma::Coherent +pub struct Bound; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Bound {} + impl Sealed for super::BoundInternal {} + impl<'a> Sealed for super::Core<'a> {} + impl<'a> Sealed for super::CoreInternal<'a> {} + impl Sealed for super::Normal {} +} + +impl DeviceContext for Bound {} +impl DeviceContext for BoundInternal {} +impl<'a> DeviceContext for Core<'a> {} +impl<'a> DeviceContext for CoreInternal<'a> {} +impl DeviceContext for Normal {} + +/// Marker trait for [`DeviceContext`] types that have internal bound-level access. +/// +/// This trait is implemented by [`CoreInternal`] and [`BoundInternal`], allowing methods that +/// require internal bus abstraction access to a bound device to be generic over both contexts. +/// +/// Methods bounded by this trait are available to bus abstractions but not to drivers. +pub trait InternalBoundContext: DeviceContext {} +impl<'a> InternalBoundContext for CoreInternal<'a> {} +impl InternalBoundContext for BoundInternal {} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &Device { + self + } +} + +/// Convert device references to bus device references. +/// +/// Bus devices can implement this trait to allow abstractions to provide the bus device in +/// class device callbacks. +/// +/// This must not be used by drivers and is intended for bus and class device abstractions only. +/// +/// # Safety +/// +/// `AsBusDevice::OFFSET` must be the offset of the embedded base `struct device` field within a +/// bus device structure. +pub unsafe trait AsBusDevice: AsRef> { + /// The relative offset to the device field. + /// + /// Use `offset_of!(bindings, field)` macro to avoid breakage. + const OFFSET: usize; + + /// Convert a reference to [`Device`] into `Self`. + /// + /// # Safety + /// + /// `dev` must be contained in `Self`. + unsafe fn from_device(dev: &Device) -> &Self + where + Self: Sized, + { + let raw = dev.as_raw(); + // SAFETY: `raw - Self::OFFSET` is guaranteed by the safety requirements + // to be a valid pointer to `Self`. + unsafe { &*raw.byte_sub(Self::OFFSET).cast::() } + } +} + +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_deref { + (unsafe { $device:ident, <$lt:lifetime> $src:ty => $dst:ty }) => { + impl<$lt> ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; + (unsafe { $device:ident, $src:ty => $dst:ty }) => { + impl ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; +} + +/// Implement [`core::ops::Deref`] traits for allowed [`DeviceContext`] conversions of a (bus +/// specific) device. +/// +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[macro_export] +macro_rules! impl_device_context_deref { + (unsafe { $device:ident }) => { + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + <'a> $crate::device::CoreInternal<'a> => $crate::device::Core<'a> + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + <'a> $crate::device::Core<'a> => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::BoundInternal => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::Bound => $crate::device::Normal + }); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_into_aref { + (<$lt:lifetime> $src:ty, $device:tt) => { + impl<$lt> ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; + ($src:ty, $device:tt) => { + impl ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; +} + +/// Implement [`core::convert::From`], such that all `&Device` can be converted to an +/// `ARef`. +#[macro_export] +macro_rules! impl_device_context_into_aref { + ($device:tt) => { + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::CoreInternal<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::Core<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!($crate::device::BoundInternal, $device); + ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! dev_printk { + ($method:ident, $dev:expr, $($f:tt)*) => { + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) + } +} + +/// Prints an emergency-level message (level 0) prefixed with device information. +/// +/// This level should be used if the system is unusable. +/// +/// Equivalent to the kernel's `dev_emerg` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_emerg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_emerg { + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) } +} + +/// Prints an alert-level message (level 1) prefixed with device information. +/// +/// This level should be used if action must be taken immediately. +/// +/// Equivalent to the kernel's `dev_alert` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_alert!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_alert { + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) } +} + +/// Prints a critical-level message (level 2) prefixed with device information. +/// +/// This level should be used in critical conditions. +/// +/// Equivalent to the kernel's `dev_crit` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_crit!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_crit { + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) } +} + +/// Prints an error-level message (level 3) prefixed with device information. +/// +/// This level should be used in error conditions. +/// +/// Equivalent to the kernel's `dev_err` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_err!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_err { + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) } +} + +/// Prints a warning-level message (level 4) prefixed with device information. +/// +/// This level should be used in warning conditions. +/// +/// Equivalent to the kernel's `dev_warn` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_warn!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_warn { + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) } +} + +/// Prints a notice-level message (level 5) prefixed with device information. +/// +/// This level should be used in normal but significant conditions. +/// +/// Equivalent to the kernel's `dev_notice` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_notice!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_notice { + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) } +} + +/// Prints an info-level message (level 6) prefixed with device information. +/// +/// This level should be used for informational messages. +/// +/// Equivalent to the kernel's `dev_info` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_info!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_info { + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) } +} + +/// Prints a debug-level message (level 7) prefixed with device information. +/// +/// This level should be used for debug messages. +/// +/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_dbg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_dbg { + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) } +} diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs new file mode 100644 index 000000000..5aead835f --- /dev/null +++ b/rust/kernel/device/property.rs @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Unified device property interface. +//! +//! C header: [`include/linux/property.h`](srctree/include/linux/property.h) + +use core::{mem::MaybeUninit, ptr}; + +use super::private::Sealed; +use crate::{ + alloc::KVec, + bindings, + error::{to_result, Result}, + fmt, + prelude::*, + str::{CStr, CString}, + sync::aref::ARef, + types::Opaque, +}; + +/// A reference-counted fwnode_handle. +/// +/// This structure represents the Rust abstraction for a +/// C `struct fwnode_handle`. This implementation abstracts the usage of an +/// already existing C `struct fwnode_handle` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A `FwNode` instance represents a valid `struct fwnode_handle` created by the +/// C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to +/// `fwnode_handle_get` ensures that the allocation remains valid at least until +/// the matching call to `fwnode_handle_put`. +#[repr(transparent)] +pub struct FwNode(Opaque); + +impl FwNode { + /// # Safety + /// + /// Callers must ensure that: + /// - The reference count was incremented at least once. + /// - They relinquish that increment. That is, if there is only one + /// increment, callers must not use the underlying object anymore -- it is + /// only safe to do so via the newly created `ARef`. + unsafe fn from_raw(raw: *mut bindings::fwnode_handle) -> ARef { + // SAFETY: As per the safety requirements of this function: + // - `NonNull::new_unchecked`: + // - `raw` is not null. + // - `ARef::from_raw`: + // - `raw` has an incremented refcount. + // - that increment is relinquished, i.e. it won't be decremented + // elsewhere. + // CAST: It is safe to cast from a `*mut fwnode_handle` to + // `*mut FwNode`, because `FwNode` is defined as a + // `#[repr(transparent)]` wrapper around `fwnode_handle`. + unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(raw.cast())) } + } + + /// Obtain the raw `struct fwnode_handle *`. + pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + self.0.get() + } + + /// Returns `true` if `&self` is an OF node, `false` otherwise. + pub fn is_of_node(&self) -> bool { + // SAFETY: The type invariant of `Self` guarantees that `self.as_raw() is a pointer to a + // valid `struct fwnode_handle`. + unsafe { bindings::is_of_node(self.as_raw()) } + } + + /// Returns an object that implements [`Display`](fmt::Display) for + /// printing the name of a node. + /// + /// This is an alternative to the default `Display` implementation, which + /// prints the full path. + pub fn display_name(&self) -> impl fmt::Display + '_ { + struct FwNodeDisplayName<'a>(&'a FwNode); + + impl fmt::Display for FwNodeDisplayName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: `self` is valid by its type invariant. + let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) }; + if name.is_null() { + return Ok(()); + } + // SAFETY: + // - `fwnode_get_name` returns null or a valid C string. + // - `name` was checked to be non-null. + let name = unsafe { CStr::from_char_ptr(name) }; + fmt::Display::fmt(name, f) + } + } + + FwNodeDisplayName(self) + } + + /// Checks if property is present or not. + pub fn property_present(&self, name: &CStr) -> bool { + // SAFETY: By the invariant of `CStr`, `name` is null-terminated. + unsafe { bindings::fwnode_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + } + + /// Returns firmware property `name` boolean value. + pub fn property_read_bool(&self, name: &CStr) -> bool { + // SAFETY: + // - `name` is non-null and null-terminated. + // - `self.as_raw()` is valid because `self` is valid. + unsafe { bindings::fwnode_property_read_bool(self.as_raw(), name.as_char_ptr()) } + } + + /// Returns the index of matching string `match_str` for firmware string + /// property `name`. + pub fn property_match_string(&self, name: &CStr, match_str: &CStr) -> Result { + // SAFETY: + // - `name` and `match_str` are non-null and null-terminated. + // - `self.as_raw` is valid because `self` is valid. + let ret = unsafe { + bindings::fwnode_property_match_string( + self.as_raw(), + name.as_char_ptr(), + match_str.as_char_ptr(), + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + + /// Returns firmware property `name` integer array values in a [`KVec`]. + pub fn property_read_array_vec<'fwnode, 'name, T: PropertyInt>( + &'fwnode self, + name: &'name CStr, + len: usize, + ) -> Result>> { + let mut val: KVec = KVec::with_capacity(len, GFP_KERNEL)?; + + let res = T::read_array_from_fwnode_property(self, name, val.spare_capacity_mut()); + let res = match res { + Ok(_) => { + // SAFETY: + // - `len` is equal to `val.capacity - val.len`, because + // `val.capacity` is `len` and `val.len` is zero. + // - All elements within the interval [`0`, `len`) were initialized + // by `read_array_from_fwnode_property`. + unsafe { val.inc_len(len) } + Ok(val) + } + Err(e) => Err(e), + }; + Ok(PropertyGuard { + inner: res, + fwnode: self, + name, + }) + } + + /// Returns integer array length for firmware property `name`. + pub fn property_count_elem(&self, name: &CStr) -> Result { + T::read_array_len_from_fwnode_property(self, name) + } + + /// Returns the value of firmware property `name`. + /// + /// This method is generic over the type of value to read. The types that + /// can be read are strings, integers and arrays of integers. + /// + /// Reading a [`KVec`] of integers is done with the separate + /// method [`Self::property_read_array_vec`], because it takes an + /// additional `len` argument. + /// + /// Reading a boolean is done with the separate method + /// [`Self::property_read_bool`], because this operation is infallible. + /// + /// For more precise documentation about what types can be read, see + /// the [implementors of Property][Property#implementors] and [its + /// implementations on foreign types][Property#foreign-impls]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{device::{Device, property::FwNode}, str::CString}; + /// fn examples(dev: &Device) -> Result { + /// let fwnode = dev.fwnode().ok_or(ENOENT)?; + /// let b: u32 = fwnode.property_read(c"some-number").required_by(dev)?; + /// if let Some(s) = fwnode.property_read::(c"some-str").optional() { + /// // ... + /// } + /// Ok(()) + /// } + /// ``` + pub fn property_read<'fwnode, 'name, T: Property>( + &'fwnode self, + name: &'name CStr, + ) -> PropertyGuard<'fwnode, 'name, T> { + PropertyGuard { + inner: T::read_from_fwnode_property(self, name), + fwnode: self, + name, + } + } + + /// Returns first matching named child node handle. + pub fn get_child_by_name(&self, name: &CStr) -> Option> { + // SAFETY: `self` and `name` are valid by their type invariants. + let child = + unsafe { bindings::fwnode_get_named_child_node(self.as_raw(), name.as_char_ptr()) }; + if child.is_null() { + return None; + } + // SAFETY: + // - `fwnode_get_named_child_node` returns a pointer with its refcount + // incremented. + // - That increment is relinquished, i.e. the underlying object is not + // used anymore except via the newly created `ARef`. + Some(unsafe { Self::from_raw(child) }) + } + + /// Returns an iterator over a node's children. + pub fn children<'a>(&'a self) -> impl Iterator> + 'a { + let mut prev: Option> = None; + + core::iter::from_fn(move || { + let prev_ptr = match prev.take() { + None => ptr::null_mut(), + Some(prev) => { + // We will pass `prev` to `fwnode_get_next_child_node`, + // which decrements its refcount, so we use + // `ARef::into_raw` to avoid decrementing the refcount + // twice. + let prev = ARef::into_raw(prev); + prev.as_ptr().cast() + } + }; + // SAFETY: + // - `self.as_raw()` is valid by its type invariant. + // - `prev_ptr` may be null, which is allowed and corresponds to + // getting the first child. Otherwise, `prev_ptr` is valid, as it + // is the stored return value from the previous invocation. + // - `prev_ptr` has its refount incremented. + // - The increment of `prev_ptr` is relinquished, i.e. the + // underlying object won't be used anymore. + let next = unsafe { bindings::fwnode_get_next_child_node(self.as_raw(), prev_ptr) }; + if next.is_null() { + return None; + } + // SAFETY: + // - `next` is valid because `fwnode_get_next_child_node` returns a + // pointer with its refcount incremented. + // - That increment is relinquished, i.e. the underlying object + // won't be used anymore, except via the newly created + // `ARef`. + let next = unsafe { FwNode::from_raw(next) }; + prev = Some(next.clone()); + Some(next) + }) + } + + /// Finds a reference with arguments. + pub fn property_get_reference_args( + &self, + prop: &CStr, + nargs: NArgs<'_>, + index: u32, + ) -> Result { + let mut out_args = FwNodeReferenceArgs::default(); + + let (nargs_prop, nargs) = match nargs { + NArgs::Prop(nargs_prop) => (nargs_prop.as_char_ptr(), 0), + NArgs::N(nargs) => (ptr::null(), nargs), + }; + + // SAFETY: + // - `self.0.get()` is valid. + // - `prop.as_char_ptr()` is valid and zero-terminated. + // - `nargs_prop` is valid and zero-terminated if `nargs` + // is zero, otherwise it is allowed to be a null-pointer. + // - The function upholds the type invariants of `out_args`, + // namely: + // - It may fill the field `fwnode` with a valid pointer, + // in which case its refcount is incremented. + // - It may modify the field `nargs`, in which case it + // initializes at least as many elements in `args`. + let ret = unsafe { + bindings::fwnode_property_get_reference_args( + self.0.get(), + prop.as_char_ptr(), + nargs_prop, + nargs, + index, + &mut out_args.0, + ) + }; + to_result(ret)?; + + Ok(out_args) + } +} + +/// The number of arguments to request [`FwNodeReferenceArgs`]. +pub enum NArgs<'a> { + /// The name of the property of the reference indicating the number of + /// arguments. + Prop(&'a CStr), + /// The known number of arguments. + N(u32), +} + +/// The return value of [`FwNode::property_get_reference_args`]. +/// +/// This structure represents the Rust abstraction for a C +/// `struct fwnode_reference_args` which was initialized by the C side. +/// +/// # Invariants +/// +/// If the field `fwnode` is valid, it owns an increment of its refcount. +/// +/// The field `args` contains at least as many initialized elements as indicated +/// by the field `nargs`. +#[repr(transparent)] +#[derive(Default)] +pub struct FwNodeReferenceArgs(bindings::fwnode_reference_args); + +impl Drop for FwNodeReferenceArgs { + fn drop(&mut self) { + if !self.0.fwnode.is_null() { + // SAFETY: + // - By the type invariants of `FwNodeReferenceArgs`, its field + // `fwnode` owns an increment of its refcount. + // - That increment is relinquished. The underlying object won't be + // used anymore because we are dropping it. + let _ = unsafe { FwNode::from_raw(self.0.fwnode) }; + } + } +} + +impl FwNodeReferenceArgs { + /// Returns the slice of reference arguments. + pub fn as_slice(&self) -> &[u64] { + // SAFETY: As per the safety invariant of `FwNodeReferenceArgs`, `nargs` + // is the minimum number of elements in `args` that is valid. + unsafe { core::slice::from_raw_parts(self.0.args.as_ptr(), self.0.nargs as usize) } + } + + /// Returns the number of reference arguments. + pub fn len(&self) -> usize { + self.0.nargs as usize + } + + /// Returns `true` if there are no reference arguments. + pub fn is_empty(&self) -> bool { + self.0.nargs == 0 + } +} + +impl fmt::Debug for FwNodeReferenceArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +// SAFETY: Instances of `FwNode` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for FwNode { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the + // refcount is non-zero. + unsafe { bindings::fwnode_handle_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is + // non-zero. + unsafe { bindings::fwnode_handle_put(obj.cast().as_ptr()) } + } +} + +enum Node<'a> { + Borrowed(&'a FwNode), + Owned(ARef), +} + +impl fmt::Display for FwNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The logic here is the same as the one in lib/vsprintf.c + // (fwnode_full_name_string). + + // SAFETY: `self.as_raw()` is valid by its type invariant. + let num_parents = unsafe { bindings::fwnode_count_parents(self.as_raw()) }; + + for depth in (0..=num_parents).rev() { + let fwnode = if depth == 0 { + Node::Borrowed(self) + } else { + // SAFETY: `self.as_raw()` is valid. + let ptr = unsafe { bindings::fwnode_get_nth_parent(self.as_raw(), depth) }; + // SAFETY: + // - The depth passed to `fwnode_get_nth_parent` is + // within the valid range, so the returned pointer is + // not null. + // - The reference count was incremented by + // `fwnode_get_nth_parent`. + // - That increment is relinquished to + // `FwNode::from_raw`. + Node::Owned(unsafe { FwNode::from_raw(ptr) }) + }; + // Take a reference to the owned or borrowed `FwNode`. + let fwnode: &FwNode = match &fwnode { + Node::Borrowed(f) => f, + Node::Owned(f) => f, + }; + + // SAFETY: `fwnode` is valid by its type invariant. + let prefix = unsafe { bindings::fwnode_get_name_prefix(fwnode.as_raw()) }; + if !prefix.is_null() { + // SAFETY: `fwnode_get_name_prefix` returns null or a + // valid C string. + let prefix = unsafe { CStr::from_char_ptr(prefix) }; + fmt::Display::fmt(prefix, f)?; + } + fmt::Display::fmt(&fwnode.display_name(), f)?; + } + + Ok(()) + } +} + +/// Implemented for types that can be read as properties. +/// +/// This is implemented for strings, integers and arrays of integers. It's used +/// to make [`FwNode::property_read`] generic over the type of property being +/// read. There are also two dedicated methods to read other types, because they +/// require more specialized function signatures: +/// - [`property_read_bool`](FwNode::property_read_bool) +/// - [`property_read_array_vec`](FwNode::property_read_array_vec) +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait Property: Sized + Sealed { + /// Used to make [`FwNode::property_read`] generic. + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result; +} + +impl Sealed for CString {} + +impl Property for CString { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let mut str: *mut u8 = ptr::null_mut(); + let pstr: *mut _ = &mut str; + + // SAFETY: + // - `name` is non-null and null-terminated. + // - `fwnode.as_raw` is valid because `fwnode` is valid. + let ret = unsafe { + bindings::fwnode_property_read_string(fwnode.as_raw(), name.as_char_ptr(), pstr.cast()) + }; + to_result(ret)?; + + // SAFETY: + // - `pstr` is a valid pointer to a NUL-terminated C string. + // - It is valid for at least as long as `fwnode`, but it's only used + // within the current function. + // - The memory it points to is not mutated during that time. + let str = unsafe { CStr::from_char_ptr(*pstr) }; + Ok(str.try_into()?) + } +} + +/// Implemented for all integers that can be read as properties. +/// +/// This helper trait is needed on top of the existing [`Property`] +/// trait to associate the integer types of various sizes with their +/// corresponding `fwnode_property_read_*_array` functions. +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait PropertyInt: Copy + Sealed { + /// Reads a property array. + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit], + ) -> Result<&'a mut [Self]>; + + /// Reads the length of a property array. + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result; +} +// This macro generates implementations of the traits `Property` and +// `PropertyInt` for integers of various sizes. Its input is a list +// of pairs separated by commas. The first element of the pair is the +// type of the integer, the second one is the name of its corresponding +// `fwnode_property_read_*_array` function. +macro_rules! impl_property_for_int { + ($($int:ty: $f:ident),* $(,)?) => { $( + impl Sealed for $int {} + impl Sealed for [$int; N] {} + + impl PropertyInt for $int { + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit], + ) -> Result<&'a mut [Self]> { + // SAFETY: + // - `fwnode`, `name` and `out` are all valid by their type + // invariants. + // - `out.len()` is a valid bound for the memory pointed to by + // `out.as_mut_ptr()`. + // CAST: It's ok to cast from `*mut MaybeUninit<$int>` to a + // `*mut $int` because they have the same memory layout. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + out.as_mut_ptr().cast(), + out.len(), + ) + }; + to_result(ret)?; + // SAFETY: Transmuting from `&'a mut [MaybeUninit]` to + // `&'a mut [Self]` is sound, because the previous call to a + // `fwnode_property_read_*_array` function (which didn't fail) + // fully initialized the slice. + Ok(unsafe { core::mem::transmute::<&mut [MaybeUninit], &mut [Self]>(out) }) + } + + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + // SAFETY: + // - `fwnode` and `name` are valid by their type invariants. + // - It's ok to pass a null pointer to the + // `fwnode_property_read_*_array` functions if `nval` is zero. + // This will return the length of the array. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + ptr::null_mut(), + 0, + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + } + + impl Property for $int { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let val: [_; 1] = <[$int; 1]>::read_from_fwnode_property(fwnode, name)?; + Ok(val[0]) + } + } + + impl Property for [$int; N] { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let mut val: [MaybeUninit<$int>; N] = [const { MaybeUninit::uninit() }; N]; + + <$int>::read_array_from_fwnode_property(fwnode, name, &mut val)?; + + // SAFETY: `val` is always initialized when + // `fwnode_property_read_*_array` is successful. + Ok(val.map(|v| unsafe { v.assume_init() })) + } + } + )* }; +} +impl_property_for_int! { + u8: fwnode_property_read_u8_array, + u16: fwnode_property_read_u16_array, + u32: fwnode_property_read_u32_array, + u64: fwnode_property_read_u64_array, + i8: fwnode_property_read_u8_array, + i16: fwnode_property_read_u16_array, + i32: fwnode_property_read_u32_array, + i64: fwnode_property_read_u64_array, +} + +/// A helper for reading device properties. +/// +/// Use [`Self::required_by`] if a missing property is considered a bug and +/// [`Self::optional`] otherwise. +/// +/// For convenience, [`Self::or`] and [`Self::or_default`] are provided. +pub struct PropertyGuard<'fwnode, 'name, T> { + /// The result of reading the property. + inner: Result, + /// The fwnode of the property, used for logging in the "required" case. + fwnode: &'fwnode FwNode, + /// The name of the property, used for logging in the "required" case. + name: &'name CStr, +} + +impl PropertyGuard<'_, '_, T> { + /// Access the property, indicating it is required. + /// + /// If the property is not present, the error is automatically logged. If a + /// missing property is not an error, use [`Self::optional`] instead. The + /// device is required to associate the log with it. + pub fn required_by(self, dev: &super::Device) -> Result { + if self.inner.is_err() { + dev_err!( + dev, + "{}: property '{}' is missing\n", + self.fwnode, + self.name + ); + } + self.inner + } + + /// Access the property, indicating it is optional. + /// + /// In contrast to [`Self::required_by`], no error message is logged if + /// the property is not present. + pub fn optional(self) -> Option { + self.inner.ok() + } + + /// Access the property or the specified default value. + /// + /// Do not pass a sentinel value as default to detect a missing property. + /// Use [`Self::required_by`] or [`Self::optional`] instead. + pub fn or(self, default: T) -> T { + self.inner.unwrap_or(default) + } +} + +impl PropertyGuard<'_, '_, T> { + /// Access the property or a default value. + /// + /// Use [`Self::or`] to specify a custom default value. + pub fn or_default(self) -> T { + self.inner.unwrap_or_default() + } +} diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs new file mode 100644 index 000000000..c81fca5b4 --- /dev/null +++ b/rust/kernel/device_id.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic implementation of device IDs. +//! +//! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is +//! expected to implement [`RawDeviceId`]. + +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + +/// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that `Self` is layout-compatible with [`RawDeviceId::RawType`]; +/// i.e. it's safe to transmute to `RawType`. +/// +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. +/// +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; +} + +/// Extension trait for [`RawDeviceId`] for devices that embed an index or context value. +/// +/// This is typically used when the device ID struct includes a field like `driver_data` +/// that is used to store a pointer-sized value (e.g., an index or context pointer). +/// +/// # Safety +/// +/// Implementers must ensure that `DRIVER_DATA_OFFSET` is the correct offset (in bytes) to +/// the context/data field (e.g., the `driver_data` field) within the raw device ID structure. +/// This field must be correctly sized to hold a `usize`. +/// +/// Ideally, the data should be added during `Self` to `RawType` conversion, +/// but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceIdIndex: RawDeviceId { + /// The offset (in bytes) to the context/data field in the raw device ID. + const DRIVER_DATA_OFFSET: usize; + + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`. + unsafe fn info_unchecked(&self) -> &'static U { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, this is stored in `IdArray` so is + // convertible to `&'static U`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<&U>() + .read() + } + } + + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`, or has NULL (or 0) as driver data. + unsafe fn info_unchecked_opt(&self) -> Option<&'static U> { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, if this is stored in `IdArray`, this is + // convertible to `Option<&'static U>`. Otherwise it is NULL which is `None` as + // `Option<&U>`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::>() + .read() + } + } +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + // This is `MaybeUninit` so any bytes inside it can carry provenance in CTFE. + // If this were `T::RawType`, integer fields would not be able to contain pointers. + ids: [MaybeUninit; N], + sentinel: MaybeUninit, + phantom: PhantomData<&'static U>, +} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be sent between threads safely +// if `&U` can. +unsafe impl Send for IdArray {} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be shared between threads safely +// if `&U` can. +unsafe impl Sync for IdArray {} + +impl IdArray { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, &'static U); N]) -> Self { + let mut raw_ids = [const { MaybeUninit::::uninit() }; N]; + + let mut i = 0usize; + while i < N { + // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is + // layout-wise compatible with `RawType`. + raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, this would be effectively + // `raw_ids[i].driver_data = ids[i].1;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(T::DRIVER_DATA_OFFSET) + .cast::<&U>() + .write(ids[i].1); + } + + i += 1; + } + + core::mem::forget(ids); + + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } + } +} + +impl IdArray { + /// Creates a new instance of the array without writing index values. + /// + /// The contents are derived from the given identifiers and context information. + /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. + pub const fn new_without_index(ids: [T; N]) -> Self { + // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. + let raw_ids: [MaybeUninit; N] = unsafe { core::mem::transmute_copy(&ids) }; + core::mem::forget(ids); + + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } + } +} + +/// A device id table. +/// +/// This trait is only implemented by `IdArray`. +/// +/// The purpose of this trait is to allow `&'static dyn IdArray` to be in context when `N` in +/// `IdArray` doesn't matter. +pub trait IdTable { + /// Obtain the pointer to the ID table. + fn as_ptr(&self) -> *const T::RawType; +} + +impl IdTable for IdArray { + fn as_ptr(&self) -> *const T::RawType { + // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance + // to access the sentinel. + core::ptr::from_ref(self).cast() + } +} + +/// Create device table alias for modpost. +#[macro_export] +macro_rules! module_device_table { + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, $id_info_type: ty, + [$(($id: expr, $info:expr $(,)?)),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + $id_info_type, + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new([$(($id, &$info),)*]); + }; + + // Case for no ID info. + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, @none, + [$($id: expr),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + (), + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new_without_index([$($id),*]); + }; +} diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs new file mode 100644 index 000000000..d2924aaae --- /dev/null +++ b/rust/kernel/devres.rs @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devres abstraction +//! +//! [`Devres`] represents an abstraction for the kernel devres (device resource management) +//! implementation. + +use crate::{ + alloc::Flags, + bindings, + device::{ + Bound, + Device, // + }, + error::to_result, + prelude::*, + revocable::{ + Revocable, + RevocableGuard, // + }, + sync::{ + aref::ARef, + rcu, + Arc, + Completion, // + }, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // + }, +}; + +/// Inner type that embeds a `struct devres_node` and the `Revocable`. +#[repr(C)] +#[pin_data] +struct Inner { + #[pin] + node: Opaque, + #[pin] + data: Revocable, + #[pin] + revocation: Completion, +} + +/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to +/// manage their lifetime. +/// +/// [`Device`] bound resources should be freed when either the resource goes out of scope or the +/// [`Device`] is unbound respectively, depending on what happens first. In any case, it is always +/// guaranteed that revoking the device resource is completed before the corresponding [`Device`] +/// is unbound. +/// +/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the +/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). +/// +/// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource +/// anymore. +/// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// +/// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s +/// [`Drop`] implementation. +/// +/// # Examples +/// +/// ```no_run +/// # #![cfg(CONFIG_HAS_IOMEM)] +/// use kernel::{ +/// bindings, +/// device::{ +/// Bound, +/// Device, +/// }, +/// devres::Devres, +/// io::{ +/// Io, +/// IoBase, +/// Mmio, +/// MmioRaw, +/// MmioBackend, +/// PhysAddr, +/// Region, // +/// }, +/// prelude::*, +/// }; +/// use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(MmioRaw>); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. +/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; +/// } +/// } +/// +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem { +/// type Backend = MmioBackend; +/// type Target = Region; +/// +/// fn as_view(self) -> Mmio<'a, Region> { +/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. +/// unsafe { Mmio::from_raw(self.0) } +/// } +/// } +/// # fn no_run(dev: &Device) -> Result<(), Error> { +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// let devres = Devres::new(dev, iomem)?; +/// +/// let res = devres.try_access().ok_or(ENXIO)?; +/// res.write8(0x42, 0x0); +/// # Ok(()) +/// # } +/// ``` +pub struct Devres { + dev: ARef, + inner: Arc>, +} + +// Calling the FFI functions from the `base` module directly from the `Devres` impl may result in +// them being called directly from driver modules. This happens since the Rust compiler will use +// monomorphisation, so it might happen that functions are instantiated within the calling driver +// module. For now, work around this with `#[inline(never)]` helpers. +// +// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to +// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported. +mod base { + use kernel::{ + bindings, + prelude::*, // + }; + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_init( + node: *mut bindings::devres_node, + release: bindings::dr_node_release_t, + free: bindings::dr_node_free_t, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_init`. + unsafe { bindings::devres_node_init(node, release, free) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_set_node_dbginfo( + node: *mut bindings::devres_node, + name: *const c_char, + size: usize, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_set_node_dbginfo`. + unsafe { bindings::devres_set_node_dbginfo(node, name, size) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_add( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_add`. + unsafe { bindings::devres_node_add(dev, node) } + } + + #[must_use] + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_remove( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) -> bool { + // SAFETY: Safety requirements are the same as `bindings::devres_node_remove`. + unsafe { bindings::devres_node_remove(dev, node) } + } +} + +impl Devres { + /// Creates a new [`Devres`] instance of the given `data`. + /// + /// The `data` encapsulated within the returned `Devres` instance' `data` will be + /// (revoked)[`Revocable`] once the device is detached. + pub fn new(dev: &Device, data: impl PinInit) -> Result + where + Error: From, + { + let inner = Arc::pin_init::( + try_pin_init!(Inner { + node <- Opaque::ffi_init(|node: *mut bindings::devres_node| { + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_node_init( + node, + Some(Self::devres_node_release), + Some(Self::devres_node_free_node), + ) + }; + + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_set_node_dbginfo( + node, + // TODO: Use `core::any::type_name::()` once it is a `const fn`, + // such that we can convert the `&str` to a `&CStr` at compile-time. + c"Devres".as_char_ptr(), + core::mem::size_of::>(), + ) + }; + }), + data <- Revocable::new(data), + revocation <- Completion::new(), + }), + GFP_KERNEL, + )?; + + // SAFETY: + // - `dev` is a valid pointer to a bound `struct device`. + // - `node` is a valid pointer to a `struct devres_node`. + // - `devres_node_add()` is guaranteed not to call `devres_node_release()` for the entire + // lifetime of `dev`. + unsafe { base::devres_node_add(dev.as_raw(), inner.node.get()) }; + + // Take additional reference count for `devres_node_add()`. + core::mem::forget(inner.clone()); + + Ok(Self { + dev: dev.into(), + inner, + }) + } + + fn data(&self) -> &Revocable { + &self.inner.data + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_node_release( + _dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner, node) }; + + // SAFETY: `inner` is a valid `Inner` pointer. + let inner = unsafe { &*inner }; + + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_node_free_node(node: *mut bindings::devres_node) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner, node) }; + + // SAFETY: `inner` points to the entire `Inner` allocation. + drop(unsafe { Arc::from_raw(inner) }); + } + + fn remove_node(&self) -> bool { + // SAFETY: + // - `self.device().as_raw()` is a valid pointer to a bound `struct device`. + // - `self.inner.node.get()` is a valid pointer to a `struct devres_node`. + unsafe { base::devres_node_remove(self.device().as_raw(), self.inner.node.get()) } + } + + /// Return a reference of the [`Device`] this [`Devres`] instance has been created with. + pub fn device(&self) -> &Device { + &self.dev + } + + /// Obtain `&'a T`, bypassing the [`Revocable`]. + /// + /// This method allows to directly obtain a `&'a T`, bypassing the [`Revocable`], by presenting + /// a `&'a Device` of the same [`Device`] this [`Devres`] instance has been created with. + /// + /// # Errors + /// + /// An error is returned if `dev` does not match the same [`Device`] this [`Devres`] instance + /// has been created with. + /// + /// # Examples + /// + /// ```no_run + /// #![cfg(CONFIG_PCI)] + /// use kernel::{ + /// device::Core, + /// devres::Devres, + /// io::Io, + /// pci, // + /// }; + /// + /// fn from_core(dev: &pci::Device>, devres: Devres>) -> Result { + /// let bar = devres.access(dev.as_ref())?; + /// + /// let _ = bar.read32(0x0); + /// + /// // might_sleep() + /// + /// bar.write32(0x42, 0x0); + /// + /// Ok(()) + /// } + /// ``` + pub fn access<'a>(&'a self, dev: &'a Device) -> Result<&'a T> { + if self.dev.as_raw() != dev.as_raw() { + return Err(EINVAL); + } + + // SAFETY: `dev` being the same device as the device this `Devres` has been created for + // proves that `self.data` hasn't been revoked and is guaranteed to not be revoked as long + // as `dev` lives; `dev` lives at least as long as `self`. + Ok(unsafe { self.data().access() }) + } + + /// [`Devres`] accessor for [`Revocable::try_access`]. + pub fn try_access(&self) -> Option> { + self.data().try_access() + } + + /// [`Devres`] accessor for [`Revocable::try_access_with`]. + pub fn try_access_with R>(&self, f: F) -> Option { + self.data().try_access_with(f) + } + + /// [`Devres`] accessor for [`Revocable::try_access_with_guard`]. + pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> { + self.data().try_access_with_guard(guard) + } +} + +// SAFETY: `Devres` can be send to any task, if `T: Send`. +unsafe impl Send for Devres {} + +// SAFETY: `Devres` can be shared with any task, if `T: Sync`. +unsafe impl Sync for Devres {} + +impl Drop for Devres { + fn drop(&mut self) { + // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data + // anymore, hence it is safe not to wait for the grace period to finish. + if unsafe { self.data().revoke_nosync() } { + self.inner.revocation.complete_all(); + + // We revoked `self.data` before devres did, hence try to remove it. + if self.remove_node() { + // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` + // for `devres_node_add()`. Since `remove_node()` was successful, we have to drop + // this additional reference count. + drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); + } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); + } + } +} + +/// Guard returned by [`DevresLt::try_access`]. +/// +/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow +/// lifetime. +pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>); + +impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> { + type Target = F::Of<'a>; + + #[inline] + fn deref(&self) -> &Self::Target { + F::cast_ref(&*self.0) + } +} + +/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access. +/// +/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow +/// lifetime in all access methods. +/// +/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`], +/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors +/// ([`Self::access_with`], [`Self::try_access_with`]). +pub struct DevresLt(Devres>) +where + for<'a> F::Of<'a>: Send; + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Creates a new [`DevresLt`] instance of the given `data`. + /// + /// # Safety + /// + /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows + /// access until the device is unbound, which may outlast `'a`. + pub unsafe fn new<'a, E>( + dev: &'a Device, + data: impl PinInit, E>, + ) -> Result + where + Error: From, + { + // SAFETY: The caller guarantees the data is valid for the device's full bound scope. + // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical + // representation; casting the slot pointer is sound. + let data = unsafe { pin_init::cast_pin_init(data) }; + + Ok(Self(Devres::new(dev, data)?)) + } + + /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with. + #[inline] + pub fn device(&self) -> &Device { + self.0.device() + } + + /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure. + /// + /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any + /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`]. + #[inline] + pub fn access_with(&self, dev: &Device, f: G) -> Result + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.access(dev).map(f) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access_with`]. + #[inline] + pub fn try_access_with(&self, f: G) -> Option + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.data().try_access_with(f) + } +} + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`]. + /// + /// This method works like [`Devres::access`], but shortens the returned reference's lifetime + /// from `'static` to `'a` via [`CovariantForLt::cast_ref`]. + #[inline] + pub fn access<'a>(&'a self, dev: &'a Device) -> Result<&'a F::Of<'a>> { + self.0.access(dev).map(F::cast_ref) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access`]. + #[inline] + pub fn try_access(&self) -> Option> { + self.0.data().try_access().map(DevresGuard) + } +} + +/// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. +fn register_foreign

(dev: &Device, data: P) -> Result +where + P: ForeignOwnable + Send + 'static, +{ + let ptr = data.into_foreign(); + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn callback(ptr: *mut kernel::ffi::c_void) { + // SAFETY: `ptr` is the pointer to the `ForeignOwnable` leaked above and hence valid. + drop(unsafe { P::from_foreign(ptr.cast()) }); + } + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid and bound device. + // - `ptr` is a valid pointer the `ForeignOwnable` devres takes ownership of. + to_result(unsafe { + // `devm_add_action_or_reset()` also calls `callback` on failure, such that the + // `ForeignOwnable` is released eventually. + bindings::devm_add_action_or_reset(dev.as_raw(), Some(callback::

), ptr.cast()) + }) +} + +/// Encapsulate `data` in a [`KBox`] and [`Drop::drop`] `data` once `dev` is unbound. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::{ +/// device::{ +/// Bound, +/// Device, // +/// }, +/// devres, // +/// }; +/// +/// /// Registration of e.g. a class device, IRQ, etc. +/// struct Registration; +/// +/// impl Registration { +/// fn new() -> Self { +/// // register +/// +/// Self +/// } +/// } +/// +/// impl Drop for Registration { +/// fn drop(&mut self) { +/// // unregister +/// } +/// } +/// +/// fn from_bound_context(dev: &Device) -> Result { +/// devres::register(dev, Registration::new(), GFP_KERNEL) +/// } +/// ``` +pub fn register(dev: &Device, data: impl PinInit, flags: Flags) -> Result +where + T: Send + 'static, + Error: From, +{ + let data = KBox::pin_init(data, flags)?; + + register_foreign(dev, data) +} diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs new file mode 100644 index 000000000..2ce09f8e9 --- /dev/null +++ b/rust/kernel/dma.rs @@ -0,0 +1,1251 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Direct memory access (DMA). +//! +//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) + +use crate::{ + bindings, + debugfs, + device::{ + self, + Bound, + Core, // + }, + error::to_result, + fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, + prelude::*, + ptr::KnownSize, + sync::aref::ARef, + transmute::{ + AsBytes, + FromBytes, // + }, + uaccess::UserSliceWriter, // +}; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, // +}; + +/// DMA address type. +/// +/// Represents a bus address used for Direct Memory Access (DMA) operations. +/// +/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on +/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`. +/// +/// Note that this may be `u64` even on 32-bit architectures. +pub type DmaAddress = bindings::dma_addr_t; + +/// Trait to be implemented by DMA capable bus devices. +/// +/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, +/// where the underlying bus is DMA capable, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`](::kernel::platform::Device) +pub trait Device<'a>: AsRef>> { + /// Set up the device's DMA streaming addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA coherent addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA addressing capabilities. + /// + /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { + bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) + }) + } + + /// Set the maximum size of a single DMA segment the device may request. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_max_seg_size(&self, size: u32) { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this parameter. + unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) } + } +} + +/// A DMA mask that holds a bitmask with the lowest `n` bits set. +/// +/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values +/// are guaranteed to never exceed the bit width of `u64`. +/// +/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaMask(u64); + +impl DmaMask { + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, results in a build error. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::new::<0>(); + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::new::<1>(); + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::new::<64>(); + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// // Build failure. + /// // let mask_overflow = DmaMask::new::<100>(); + /// ``` + #[inline] + pub const fn new() -> Self { + let Ok(mask) = Self::try_new(N) else { + build_error!("Invalid DMA Mask."); + }; + + mask + } + + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, returns [`EINVAL`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::try_new(0)?; + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::try_new(1)?; + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::try_new(64)?; + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// let mask_overflow = DmaMask::try_new(100); + /// assert!(mask_overflow.is_err()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub const fn try_new(n: u32) -> Result { + Ok(Self(match n { + 0 => 0, + 1..=64 => u64::MAX >> (64 - n), + _ => return Err(EINVAL), + })) + } + + /// Returns the underlying `u64` bitmask value. + #[inline] + pub const fn value(&self) -> u64 { + self.0 + } +} + +/// Possible attributes associated with a DMA mapping. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`attrs`] module. +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::{Bound, Device}; +/// use kernel::dma::{attrs::*, Coherent}; +/// +/// # fn test(dev: &Device) -> Result { +/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; +/// let c: Coherent<[u64]> = +/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// # Ok::<(), Error>(()) } +/// ``` +#[derive(Clone, Copy, PartialEq)] +#[repr(transparent)] +pub struct Attrs(u32); + +impl Attrs { + /// Get the raw representation of this attribute. + pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { + self.0 as crate::ffi::c_ulong + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Attrs) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Attrs { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Attrs { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Attrs { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// DMA mapping attributes. +pub mod attrs { + use super::Attrs; + + /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads + /// and writes may pass each other. + pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); + + /// Specifies that writes to the mapping may be buffered to improve performance. + pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); + + /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming + /// that it has been already transferred to 'device' domain. + pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); + + /// Forces contiguous allocation of the buffer in physical memory. + pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); + + /// Hints DMA-mapping subsystem that it's probably not worth the time to try + /// to allocate memory to in a way that gives better TLB efficiency. + pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); + + /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to + /// `__GFP_NOWARN`). + pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); + + /// Indicates that the buffer is fully accessible at an elevated privilege level (and + /// ideally inaccessible or at least read-only at lesser-privileged levels). + pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); + + /// Indicates that the buffer is MMIO memory. + pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO); +} + +/// DMA data direction. +/// +/// Corresponds to the C [`enum dma_data_direction`]. +/// +/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum DataDirection { + /// The DMA mapping is for bidirectional data transfer. + /// + /// This is used when the buffer can be both read from and written to by the device. + /// The cache for the corresponding memory region is both flushed and invalidated. + Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL), + + /// The DMA mapping is for data transfer from memory to the device (write). + /// + /// The CPU has prepared data in the buffer, and the device will read it. + /// The cache for the corresponding memory region is flushed before device access. + ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE), + + /// The DMA mapping is for data transfer from the device to memory (read). + /// + /// The device will write data into the buffer for the CPU to read. + /// The cache for the corresponding memory region is invalidated before CPU access. + FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE), + + /// The DMA mapping is not for data transfer. + /// + /// This is primarily for debugging purposes. With this direction, the DMA mapping API + /// will not perform any cache coherency operations. + None = Self::const_cast(bindings::dma_data_direction_DMA_NONE), +} + +impl DataDirection { + /// Casts the bindgen-generated enum type to a `u32` at compile time. + /// + /// This function will cause a compile-time error if the underlying value of the + /// C enum is out of bounds for `u32`. + const fn const_cast(val: bindings::dma_data_direction) -> u32 { + // CAST: The C standard allows compilers to choose different integer types for enums. + // To safely check the value, we cast it to a wide signed integer type (`i128`) + // which can hold any standard C integer enum type without truncation. + let wide_val = val as i128; + + // Check if the value is outside the valid range for the target type `u32`. + // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison. + if wide_val < 0 || wide_val > u32::MAX as i128 { + // Trigger a compile-time error in a const context. + build_error!("C enum value is out of bounds for the target type `u32`."); + } + + // CAST: This cast is valid because the check above guarantees that `wide_val` + // is within the representable range of `u32`. + wide_val as u32 + } +} + +impl From for bindings::dma_data_direction { + /// Returns the raw representation of [`enum dma_data_direction`]. + fn from(direction: DataDirection) -> Self { + // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum. + // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible + // with the enum variants of `DataDirection`, which is a valid assumption given our + // compile-time checks. + direction as u32 as Self + } +} + +/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object. +/// +/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA +/// address is not exposed and it cannot be accessed by a device. This means it can safely be used +/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe). +/// +/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then +/// convert it into a [`Coherent`] object to share it with the device. +/// +/// # Examples +/// +/// `CoherentBox`: +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// *dmem = 42; +/// let dmem: Coherent = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +/// +/// `CoherentBox<[T]>`: +/// +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// dmem.fill(42); +/// let dmem: Coherent<[u64]> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +pub struct CoherentBox(Coherent); + +impl CoherentBox<[T]> { + /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Initializes the element at `i` using the given initializer. + /// + /// Returns `EINVAL` if `i` is out of bounds. + pub fn init_at(&mut self, i: usize, init: impl Init) -> Result + where + Error: From, + { + if i >= self.0.len() { + return Err(EINVAL); + } + + let ptr = &raw mut self[i]; + + // SAFETY: + // - `ptr` is valid, properly aligned, and within this allocation. + // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on + // error cannot leave the element in an invalid state. + // - The DMA address has not been exposed yet, so there is no concurrent device access. + unsafe { pin_init::raw_try_init(ptr, init)? }; + + Ok(()) + } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`]. + /// + /// # Examples + /// + /// ``` + /// use core::ops::Deref; + /// + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// CoherentBox + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// let c: CoherentBox<[u8]> = + /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// assert_eq!(c.deref(), &data); + /// # Ok::<(), Error>(()) } + /// ``` + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result + where + T: Copy, + { + let mut slice = Self(Coherent::::alloc_slice_with_attrs( + dev, + data.len(), + gfp_flags, + dma_attrs, + )?); + + // PANIC: `slice` was created with length `data.len()`. + slice.copy_from_slice(data); + + Ok(slice) + } + + /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } +} + +impl CoherentBox { + /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [`CoherentBox::zeroed_slice`], but for a single element. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } +} + +impl Deref for CoherentBox { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for CoherentBox { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_mut() } + } +} + +impl From> for Coherent { + #[inline] + fn from(value: CoherentBox) -> Self { + value.0 + } +} + +/// An abstraction of the `dma_alloc_coherent` API. +/// +/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map +/// large coherent DMA regions. +/// +/// A [`Coherent`] instance contains a pointer to the allocated region (in the +/// processor's virtual address space) and the device address which can be given to the device +/// as the DMA address base of the region. The region is released once [`Coherent`] +/// is dropped. +/// +/// # Invariants +/// +/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer +/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the +/// region. +/// - The size in bytes of the allocation is equal to size information via pointer. +// TODO +// +// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness +// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure +// that device resources can never survive device unbind. +// +// However, it is neither desirable nor necessary to protect the allocated memory of the DMA +// allocation from surviving device unbind; it would require RCU read side critical sections to +// access the memory, which may require subsequent unnecessary copies. +// +// Hence, find a way to revoke the device resources of a `Coherent`, but not the +// entire `Coherent` including the allocated memory itself. +pub struct Coherent { + dev: ARef, + dma_addr: DmaAddress, + cpu_addr: NonNull, + dma_attrs: Attrs, +} + +impl Coherent { + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + T::size(self.cpu_addr.as_ptr()) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.cpu_addr.as_ptr() + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + self.cpu_addr.as_ptr() + } + + /// Returns a DMA address which may be given to the device as the base of the region. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + #[inline] + pub unsafe fn as_ref(&self) -> &T { + // SAFETY: per safety requirement. + unsafe { &*self.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + #[expect(clippy::mut_from_ref, reason = "unsafe to use API")] + #[inline] + pub unsafe fn as_mut(&self) -> &mut T { + // SAFETY: per safety requirement. + unsafe { &mut *self.as_mut_ptr() } + } +} + +impl Coherent { + /// Allocates a region of `T` of coherent memory. + fn alloc_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + let mut dma_addr = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + core::mem::size_of::(), + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?; + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for `T`, + // hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_addr, + cpu_addr, + dma_attrs, + }) + } + + /// Allocates a region of type `T` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64; 4]> = + /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } + + /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is + /// initialized with `init`. + pub fn init_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + init: impl Init, + ) -> Result + where + Error: From, + { + let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?; + let ptr = dmem.as_mut_ptr(); + + // SAFETY: + // - `ptr` is valid, properly aligned, and points to exclusively owned memory. + // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying + // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` + // requirements we are bypassing. + unsafe { pin_init::raw_try_init(ptr, init)? }; + + Ok(dmem) + } + + /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized + /// with `init`. + #[inline] + pub fn init( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + init: impl Init, + ) -> Result + where + Error: From, + { + Self::init_with_attrs(dev, gfp_flags, Attrs(0), init) + } + + /// Allocates a region of `[T; len]` of coherent memory. + fn alloc_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + // `dma_alloc_attrs` cannot handle zero-length allocation, bail early. + if len == 0 { + Err(EINVAL)?; + } + + let size = core::mem::size_of::().checked_mul(len).ok_or(ENOMEM)?; + let mut dma_addr = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len); + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for + // `[T; len]`, hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Coherent { + dev: dev.into(), + dma_addr, + cpu_addr, + dma_attrs, + }) + } + + /// Allocates a zeroed region of type `T` of coherent memory. + /// + /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::::zeroed_slices` support + /// a runtime length. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64]> = + /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) + } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// // `c` has the same content as `data`. + /// let c: Coherent<[u8]> = + /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> + where + T: Copy, + { + CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into) + } + + /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } +} + +impl Coherent<[T]> { + /// Returns the number of elements `T` in this allocation. + /// + /// Note that this is not the size of the allocation in bytes, which is provided by + /// [`Self::size`]. + #[inline] + #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")] + pub fn len(&self) -> usize { + self.cpu_addr.len() + } +} + +/// Note that the device configured to do DMA must be halted before this object is dropped. +impl Drop for Coherent { + fn drop(&mut self) { + let size = T::size(self.cpu_addr.as_ptr()); + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // The cpu address, and the dma address are valid due to the type invariants on + // `Coherent`. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + size, + self.cpu_addr.as_ptr().cast(), + self.dma_addr, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: It is safe to send a `Coherent` to another thread if `T` +// can be sent to another thread. +unsafe impl Send for Coherent {} + +// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all +// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, +// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. +// The safe methods only return metadata or raw pointers whose use requires `unsafe`. +unsafe impl Sync for Coherent {} + +impl debugfs::BinaryWriter for Coherent { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + // If the offset is too large for a usize (e.g. on 32-bit platforms), + // then consider that as past EOF and just return 0 bytes. + let Ok(offset_val) = usize::try_from(*offset) else { + return Ok(0); + }; + + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); + + writer.write_dma(self, offset_val, count)?; + + *offset += count as i64; + Ok(count) + } +} + +/// An opaque DMA allocation without a kernel virtual mapping. +/// +/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory. +/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel +/// virtual mapping is created for the buffer. The value returned by the C API as the CPU +/// address is an opaque handle used only to free the allocation. +/// +/// This is useful for buffers that are only ever accessed by hardware. +/// +/// # Invariants +/// +/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with +/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. +/// - `dma_addr` is the corresponding bus address for device DMA. +/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. +/// - `dma_attrs` contains the attributes used for the allocation, always including +/// `DMA_ATTR_NO_KERNEL_MAPPING`. +pub struct CoherentHandle { + dev: ARef, + dma_addr: DmaAddress, + cpu_handle: NonNull, + size: usize, + dma_attrs: Attrs, +} + +impl CoherentHandle { + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + /// + /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is + /// always set implicitly. + /// + /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. + pub fn alloc_with_attrs( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + if size == 0 { + return Err(EINVAL); + } + + let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + let mut dma_addr = 0; + // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. + let cpu_handle = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + + let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; + + // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, + // and we hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_addr, + cpu_handle, + size, + dma_attrs, + }) + } + + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + #[inline] + pub fn alloc( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) + } + + /// Returns the DMA address for this allocation. + /// + /// This address can be programmed into device hardware for DMA access. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for CoherentHandle { + fn drop(&mut self) { + // SAFETY: All values are valid by the type invariants on `CoherentHandle`. + // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + self.size, + self.cpu_handle.as_ptr(), + self.dma_addr, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, +// and a size. None of these are tied to a specific thread. +unsafe impl Send for CoherentHandle {} + +// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only +// operations on `&CoherentHandle` are reading the DMA address and size, both of which are +// plain `Copy` values. +unsafe impl Sync for CoherentHandle {} + +/// View type for `Coherent`. +/// +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA +/// address. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_addr: DmaAddress, +} + +impl Copy for CoherentView<'_, T> {} +impl Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns the DMA address which may be given to the device as base of the region. + #[inline] + pub fn dma_address(self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_addr = view.dma_addr + offset as DmaAddress; + CoherentView { + dma_addr, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl IoCapable for CoherentIoBackend +where + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_addr: self.dma_addr, + } + } +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 000000000..c9c74c4dd --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! This documentation describes how to implement a bus specific driver API and how to align it with +//! the design of (bus specific) devices. +//! +//! Note: Readers are expected to know the content of the documentation of [`Device`] and +//! [`DeviceContext`]. +//! +//! # Driver Trait +//! +//! The main driver interface is defined by a bus specific driver trait. For instance: +//! +//! ```ignore +//! pub trait Driver { +//! /// The type holding information about each device ID supported by the driver. +//! type IdInfo: 'static; +//! +//! /// The type of the driver's bus device private data. +//! type Data<'bound>: Send + 'bound; +//! +//! /// The table of OF device ids supported by the driver. +//! const OF_ID_TABLE: Option> = None; +//! +//! /// The table of ACPI device ids supported by the driver. +//! const ACPI_ID_TABLE: Option> = None; +//! +//! /// Driver probe. +//! fn probe<'bound>( +//! dev: &'bound Device>, +//! id_info: &'bound Self::IdInfo, +//! ) -> impl PinInit, Error> + 'bound; +//! +//! /// Driver unbind (optional). +//! fn unbind<'bound>( +//! dev: &'bound Device>, +//! this: Pin<&Self::Data<'bound>>, +//! ) { +//! let _ = (dev, this); +//! } +//! } +//! ``` +//! +//! For specific examples see: +//! +//! * [`platform::Driver`](kernel::platform::Driver) +#![cfg_attr( + CONFIG_AUXILIARY_BUS, + doc = "* [`auxiliary::Driver`](kernel::auxiliary::Driver)" +)] +#![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] +//! +//! The `probe()` callback should return a +//! `impl PinInit, Error>`, i.e. the driver's private data. The bus +//! abstraction should store the pointer in the corresponding bus device. The generic +//! [`Device`] infrastructure provides common helpers for this purpose on its +//! [`Device`] implementation. +//! +//! All driver callbacks should provide a reference to the driver's private data. Once the driver +//! is unbound from the device, the bus abstraction should take back the ownership of the driver's +//! private data from the corresponding [`Device`] and [`drop`] it. +//! +//! All driver callbacks should provide a [`Device`] reference (see also [`device::Core`]). +//! +//! # Adapter +//! +//! The adapter implementation of a bus represents the abstraction layer between the C bus +//! callbacks and the Rust bus callbacks. It therefore has to be generic over an implementation of +//! the [driver trait](#driver-trait). +//! +//! ```ignore +//! pub struct Adapter; +//! ``` +//! +//! There's a common [`Adapter`] trait that can be implemented to inherit common driver +//! infrastructure, such as finding the ID info from an [`of::IdTable`] or [`acpi::IdTable`]. +//! +//! # Driver Registration +//! +//! In order to register C driver types (such as `struct platform_driver`) the [adapter](#adapter) +//! should implement the [`RegistrationOps`] trait. +//! +//! This trait implementation can be used to create the actual registration with the common +//! [`Registration`] type. +//! +//! Typically, bus abstractions want to provide a bus specific `module_bus_driver!` macro, which +//! creates a kernel module with exactly one [`Registration`] for the bus specific adapter. +//! +//! The generic driver infrastructure provides a helper for this with the [`module_driver`] macro. +//! +//! # Device IDs +//! +//! Besides the common device ID types, such as [`of::DeviceId`] and [`acpi::DeviceId`], most buses +//! may need to implement their own device ID types. +//! +//! For this purpose the generic infrastructure in [`device_id`] should be used. +//! +//! [`Core`]: device::Core +//! [`Device`]: device::Device +//! [`Device`]: device::Device +//! [`Device`]: device::Device +//! [`DeviceContext`]: device::DeviceContext +//! [`device_id`]: kernel::device_id +//! [`module_driver`]: kernel::module_driver + +use crate::{ + acpi, + device, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; + +/// Trait describing the layout of a specific device driver. +/// +/// This trait describes the layout of a specific driver structure, such as `struct pci_driver` or +/// `struct platform_driver`. +/// +/// # Safety +/// +/// Implementors must guarantee that: +/// - `DriverType` is `repr(C)`, +/// - `DriverData` is the type of the driver's device private data. +/// - `DriverType` embeds a valid `struct device_driver` at byte offset `DEVICE_DRIVER_OFFSET`. +pub unsafe trait DriverLayout { + /// The specific driver type embedding a `struct device_driver`. + type DriverType: Default; + + /// The type of the driver's bus device private data. + type DriverData<'bound>; + + /// Byte offset of the embedded `struct device_driver` within `DriverType`. + /// + /// This must correspond exactly to the location of the embedded `struct device_driver` field. + const DEVICE_DRIVER_OFFSET: usize; +} + +/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, +/// Amba, etc.) to provide the corresponding subsystem specific implementation to register / +/// unregister a driver of the particular type (`DriverType`). +/// +/// For instance, the PCI subsystem would set `DriverType` to `bindings::pci_driver` and call +/// `bindings::__pci_register_driver` from `RegistrationOps::register` and +/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. +/// +/// # Safety +/// +/// A call to [`RegistrationOps::unregister`] for a given instance of `DriverType` is only valid if +/// a preceding call to [`RegistrationOps::register`] has been successful. +pub unsafe trait RegistrationOps: DriverLayout { + /// Registers a driver. + /// + /// # Safety + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`RegistrationOps::unregister`]. + unsafe fn register( + reg: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`RegistrationOps::register`]. + /// + /// # Safety + /// + /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for + /// the same `reg`. + unsafe fn unregister(reg: &Opaque); +} + +/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. +/// `bindings::pci_driver`). Therefore a [`Registration`] must be initialized with a type that +/// implements the [`RegistrationOps`] trait, such that the generic `T::register` and +/// `T::unregister` calls result in the subsystem specific registration calls. +/// +///Once the `Registration` structure is dropped, the driver is unregistered. +#[pin_data(PinnedDrop)] +pub struct Registration { + #[pin] + reg: Opaque, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl Sync for Registration {} + +// SAFETY: Both registration and unregistration are implemented in C and safe to be performed from +// any thread, so `Registration` is `Send`. +unsafe impl Send for Registration {} + +impl Registration { + extern "C" fn post_unbind_callback(dev: *mut bindings::device) { + // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to + // a `struct device`. + // + // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. + let dev = unsafe { &*dev.cast::>>() }; + + // `remove()` has been completed at this point; devres resources are still valid and will + // be released after the driver's bus device private data is dropped. + // + // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the + // driver's bus device private data type. + drop(unsafe { dev.drvdata_obtain::>() }); + } + + /// Attach generic `struct device_driver` callbacks. + fn callbacks_attach(drv: &Opaque) { + let ptr = drv.get().cast::(); + + // SAFETY: + // - `drv.get()` yields a valid pointer to `Self::DriverType`. + // - Adding `DEVICE_DRIVER_OFFSET` yields the address of the embedded `struct device_driver` + // as guaranteed by the safety requirements of the `Driver` trait. + let base = unsafe { ptr.add(T::DEVICE_DRIVER_OFFSET) }; + + // CAST: `base` points to the offset of the embedded `struct device_driver`. + let base = base.cast::(); + + // SAFETY: It is safe to set the fields of `struct device_driver` on initialization. + unsafe { (*base).p_cb.post_unbind_rust = Some(Self::post_unbind_callback) }; + } + + /// Creates a new instance of the registration object. + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit + where + T: 'static, + { + try_pin_init!(Self { + reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| { + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. + unsafe { ptr.write(T::DriverType::default()) }; + + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has + // just been initialised above, so it's also valid for read. + let drv = unsafe { &*(ptr as *const Opaque) }; + + Self::callbacks_attach(drv); + + // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`. + unsafe { T::register(drv, name, module) } + }), + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The existence of `self` guarantees that `self.reg` has previously been + // successfully registered with `T::register` + unsafe { T::unregister(&self.reg) }; + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + + #[$crate::prelude::pin_data] + struct DriverModule { + #[pin] + _driver: $crate::driver::Registration>, + } + + impl $crate::InPlaceModule for DriverModule { + fn init( + module: &'static $crate::ThisModule + ) -> impl ::pin_init::PinInit { + $crate::try_pin_init!(Self { + _driver <- $crate::driver::Registration::new( + ::NAME, + module, + ), + }) + } + } + + $crate::prelude::module! { + type: DriverModule, + $($f)* + } + } +} + +/// The bus independent adapter to match a drivers and a devices. +/// +/// This trait should be implemented by the bus specific adapter, which represents the connection +/// of a device and a driver. +/// +/// It provides bus independent functions for device / driver interactions. +pub trait Adapter { + /// The type holding driver private data about each device id supported by the driver. + type IdInfo: 'static; + + /// The [`acpi::IdTable`] of the corresponding driver + fn acpi_id_table() -> Option>; + + /// The [`of::IdTable`] of the corresponding driver. + fn of_id_table() -> Option>; + + /// Returns the driver's private data from the matching entry of any of the ID tables, if any. + /// + /// If this returns `None`, it means that there is no match in any of the ID tables directly + /// associated with a [`device::Device`]. + /// + /// # Safety + /// + /// The caller must ensure that the `dev` matched data is of type `Self::IdInfo`. + #[inline] + unsafe fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + // SAFETY: `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let data = unsafe { bindings::device_get_match_data(dev.as_raw()) }; + + // SAFETY: Per safety requirement, `data` is of type `Self::IdInfo`. + unsafe { data.cast::().as_ref() } + } +} diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs new file mode 100644 index 000000000..6b88ade28 --- /dev/null +++ b/rust/kernel/drm/device.rs @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM device. +//! +//! C header: [`include/drm/drm_device.h`](srctree/include/drm/drm_device.h) + +use crate::{ + alloc::allocator::Kmalloc, + bindings, + device, + drm::{ + self, + driver::AllocImpl, + private::Sealed, // + }, + error::from_err_ptr, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, + workqueue::{ + HasDelayedWork, + HasWork, + Work, + WorkItem, // + }, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, +}; + +#[cfg(CONFIG_DRM_LEGACY)] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),*, + firstopen: None, + preclose: None, + dma_ioctl: None, + dma_quiescent: None, + context_dtor: None, + irq_handler: None, + irq_preinstall: None, + irq_postinstall: None, + irq_uninstall: None, + get_vblank_counter: None, + enable_vblank: None, + disable_vblank: None, + dev_priv_size: 0, + } + } +} + +#[cfg(not(CONFIG_DRM_LEGACY))] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),* + } + } +} + +/// A trait implemented by all possible contexts a [`Device`] can be used in. +/// +/// A [`Device`] can be in one of the following contexts: +/// +/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may +/// or may not be registered with userspace. +/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl +/// dispatch context. +/// - [`Registered`]: The device is currently registered with userspace and the parent bus device +/// is bound. +/// +/// Both `Device` and `Device` dereference to `Device` ([`Normal`]), +/// so any method available on a [`Normal`] device is also available in the other contexts. +pub trait DeviceContext: Sealed + Send + Sync + 'static {} + +/// The general-purpose, reference-counted [`DeviceContext`]. +/// +/// A [`Device`] in this context may or may not be registered with userspace. This context is used +/// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. +/// +/// [`AlwaysRefCounted`] is only implemented for `Device`, making this the required +/// context for [`ARef`]-based device handles. +pub struct Normal; + +impl Sealed for Normal {} +impl DeviceContext for Normal {} + +/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace. +/// +/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is +/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a +/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// The parent bus device is bound for the duration of any reference to a `Device`. +pub struct Registered; + +impl Sealed for Registered {} +impl DeviceContext for Registered {} + +/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously. +/// +/// A [`Device`] in this context has been registered at some point, but may be concurrently +/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the +/// device remains registered for the duration of the critical section. +/// +/// # Invariants +/// +/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at +/// some point. +pub struct Ioctl; + +impl Sealed for Ioctl {} +impl DeviceContext for Ioctl {} + +/// A [`Device`] which is known at compile-time to be unregistered with userspace. +/// +/// This type allows performing operations which are only safe to do before userspace registration, +/// and can be used to create a [`Registration`](drm::driver::Registration) once the driver is ready +/// to register the device with userspace. +/// +/// Since DRM device initialization must be single-threaded, this object is not thread-safe. +/// +/// # Invariants +/// +/// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been +/// registered with userspace until this type is dropped. +pub struct UnregisteredDevice(ARef>, NotThreadSafe); + +impl Deref for UnregisteredDevice { + type Target = Device; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl UnregisteredDevice { + const fn compute_features() -> u32 { + let mut features = drm::driver::FEAT_GEM; + + if T::FEAT_RENDER { + features |= drm::driver::FEAT_RENDER; + } + + features + } + + const VTABLE: bindings::drm_driver = drm_legacy_fields! { + load: None, + open: Some(drm::File::::open_callback), + postclose: Some(drm::File::::postclose_callback), + unload: None, + release: Some(Device::::release), + master_set: None, + master_drop: None, + debugfs_init: None, + + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, + + show_fdinfo: None, + fbdev_probe: None, + + major: T::INFO.major, + minor: T::INFO.minor, + patchlevel: T::INFO.patchlevel, + name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), + desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), + + driver_features: Self::compute_features(), + ioctls: T::IOCTLS.as_ptr(), + num_ioctls: T::IOCTLS.len() as i32, + fops: &Self::GEM_FOPS, + }; + + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::().as_ptr()); + + /// Create a new `UnregisteredDevice` for a `drm::Driver`. + /// + /// This can be used to create a [`Registration`](kernel::drm::Registration). + pub fn new( + dev: &T::ParentDevice, + data: impl PinInit, + ) -> Result { + // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` + // compatible `Layout`. + let layout = Kmalloc::aligned_layout(Layout::new::>()); + + // Use a temporary vtable without a `release` callback until `data` is initialized, so + // init failure can release the DRM device without dropping uninitialized fields. + let alloc_vtable = bindings::drm_driver { + release: None, + ..Self::VTABLE + }; + + // SAFETY: + // - `alloc_vtable` reference remains valid until no longer used, + // - `dev` is valid by its type invarants, + let raw_drm: *mut Device = unsafe { + bindings::__drm_dev_alloc( + dev.as_ref().as_raw(), + &alloc_vtable, + layout.size(), + mem::offset_of!(Device, dev), + ) + } + .cast(); + let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?; + + // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was + // successful. + let drm_dev = unsafe { Device::into_drm_device(raw_drm) }; + + // SAFETY: `raw_drm` is a valid pointer to `Self`. + let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) }; + + // SAFETY: + // - `raw_data` is a valid pointer to uninitialized memory. + // - `raw_data` will not move until it is dropped. + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { + // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the + // refcount must be non-zero. + unsafe { bindings::drm_dev_put(drm_dev) }; + })?; + + // SAFETY: `drm_dev` is still private to this function. + unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; + + // SAFETY: `raw_drm` is valid; no concurrent access before registration. + unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) }; + + // SAFETY: The reference count is one, and now we take ownership of that reference as a + // `drm::Device`. + // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`. + // `Self` cannot be copied or sent to another thread - ensuring that `drm_dev_register` + // won't be called during its lifetime and that the device is unregistered. + Ok(Self(unsafe { ARef::from_raw(raw_drm) }, NotThreadSafe)) + } +} + +/// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`]. +/// +/// A device in the [`Registered`] context is currently registered with userspace and its parent +/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context. +/// +/// # Invariants +/// +/// * `self.dev` is a valid instance of a `struct device`. +/// * The data layout of `Self` remains the same across all implementations of `C`. +/// * Any invariants for `C` also apply. +#[repr(C)] +pub struct Device { + dev: Opaque, + data: T::Data, + pub(super) registration_data: UnsafeCell>>, + _ctx: PhantomData, +} + +impl Device { + pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { + self.dev.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. + unsafe fn from_drm_device(ptr: *const bindings::drm_device) -> *mut Self { + // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a + // `struct drm_device` embedded in `Self`. + unsafe { crate::container_of!(Opaque::cast_from(ptr), Self, dev) }.cast_mut() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to `Self`. + unsafe fn into_drm_device(ptr: NonNull) -> *mut bindings::drm_device { + // SAFETY: By the safety requirements of this function, `ptr` is a valid pointer to `Self`. + unsafe { &raw mut (*ptr.as_ptr()).dev }.cast() + } + + /// Not intended to be called externally, except via declare_drm_ioctls!() + /// + /// # Safety + /// + /// * Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points + /// to can't drop to zero, for the duration of this function call and the entire duration when + /// the returned reference exists. + /// * Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is + /// embedded in `Self`. + /// * Callers promise that any type invariants of `C` will be upheld. + #[doc(hidden)] + pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { + // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a + // `struct drm_device` embedded in `Self`. + let ptr = unsafe { Self::from_drm_device(ptr) }; + + // SAFETY: `ptr` is valid by the safety requirements of this function. + unsafe { &*ptr.cast() } + } + + extern "C" fn release(ptr: *mut bindings::drm_device) { + // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. + let this = unsafe { Self::from_drm_device(ptr) }; + + // SAFETY: + // - When `release` runs it is guaranteed that there is no further access to `this`. + // - `this` is valid for dropping. + unsafe { core::ptr::drop_in_place(this) }; + } + + /// Change the [`DeviceContext`] for a [`Device`]. + /// + /// # Safety + /// + /// The caller promises that `self` fulfills all of the guarantees provided by the given + /// [`DeviceContext`]. + pub(crate) unsafe fn assume_ctx(&self) -> &Device { + // SAFETY: The data layout is identical via our type invariants. + unsafe { mem::transmute(self) } + } +} + +impl Device { + /// Guard against the parent bus device being unbound. + /// + /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise. + /// + /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound. + #[must_use] + pub fn registration_guard(&self) -> Option> { + let mut idx: i32 = 0; + // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`. + if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } { + // INVARIANT: + // - `idx` is the SRCU index from the successful `drm_dev_enter()` above. + // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning + // `drm_dev_unplug()` has not completed; since it is only called from + // `Registration::drop()` during parent unbind, the parent is still bound. + Some(RegistrationGuard { + // SAFETY: See INVARIANT above; the `Registered` context invariant holds. + dev: unsafe { self.assume_ctx() }, + idx, + _not_send: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard proving the DRM device is registered and the parent bus device is bound. +/// +/// The guard dereferences to [`Device`], providing access to the DRM device with +/// the guarantee that the parent bus device is bound for the entire duration of the critical +/// section. +/// +/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call. +/// - The parent bus device of `dev` is bound for the lifetime of this guard. +#[must_use] +pub struct RegistrationGuard<'a, T: drm::Driver> { + dev: &'a Device, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Device { + /// Returns a reference to the registration data with lifetime shortened from `'static`. + /// + /// # Safety + /// + /// The returned reference must not be exposed to code that can choose a concrete lifetime for + /// it, as that would be unsound for types that are invariant over their lifetime parameter + /// (e.g. it must be passed through an HRTB-bounded closure). + #[inline] + unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> { + // SAFETY: + // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid. + // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes + // are erased at runtime. + // - Caller guarantees the reference is only used behind an HRTB, making the lifetime + // shortening sound regardless of variance. + unsafe { (*self.registration_data.get()).cast::<_>().as_ref() } + } + + /// Access the registration data through a closure, with the lifetime tied to the closure + /// scope. + /// + /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid + /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop` + /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to + /// complete. + #[inline] + pub fn registration_data_with(&self, f: F) -> R + where + F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R, + { + // SAFETY: `Registered` guarantees the device is registered and the parent bus device is + // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references + // with a concrete short lifetime, satisfying the lifetime requirement of + // `registration_data_unchecked`. + f(unsafe { self.registration_data_unchecked() }) + } +} + +impl Deref for RegistrationGuard<'_, T> { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + self.dev + } +} + +impl Drop for RegistrationGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed + // by the type invariants of `RegistrationGuard`. + unsafe { bindings::drm_dev_exit(self.idx) }; + } +} + +impl Deref for Device { + type Target = T::Data; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +// SAFETY: DRM device objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::drm_dev_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: `obj` is a valid pointer to `Self`. + let drm_dev = unsafe { Self::into_drm_device(obj) }; + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_dev_put(drm_dev) }; + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &T::ParentDevice { + // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, + // which is guaranteed by the type invariant. + let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &T::ParentDevice { + let dev = (**self).as_ref().as_ref(); + + // SAFETY: A `Device` guarantees that the parent device is bound. + let dev = unsafe { dev.as_bound() }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +// SAFETY: A `drm::Device` can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected +// by the synchronization in `struct drm_device`. +unsafe impl Sync for Device {} + +impl WorkItem for Device +where + T::Data: WorkItem>, + T::Data: HasWork, +{ + type Pointer = ARef; + + fn run(ptr: ARef) { + T::Data::run(ptr); + } +} + +// SAFETY: +// +// - `raw_get_work` and `work_container_of` return valid pointers by relying on +// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is +// stored inline in `drm::Device`, so the `container_of` call is valid. +// +// - The two methods are true inverses of each other: given `ptr: *mut +// Device`, `raw_get_work` will return a `*mut Work, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, +// `work_container_of` will return a `*mut Device` through `container_of`. +unsafe impl HasWork for Device +where + T: drm::Driver, + T::Data: HasWork, + C: DeviceContext, +{ + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work { + // SAFETY: The caller promises that `ptr` points to a valid `Device`. + let data_ptr = unsafe { &raw mut (*ptr).data }; + + // SAFETY: `data_ptr` is a valid pointer to `T::Data`. + unsafe { T::Data::raw_get_work(data_ptr) } + } + + unsafe fn work_container_of(ptr: *mut Work) -> *mut Self { + // SAFETY: The caller promises that `ptr` points at a `Work` field in + // `T::Data`. + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + + // SAFETY: `T::Data` is stored as the `data` field in `Device`. + unsafe { crate::container_of!(data_ptr, Self, data) } + } +} + +// SAFETY: Our `HasWork` implementation returns a `work_struct` that is +// stored in the `work` field of a `delayed_work` with the same access rules as +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, +// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that +// is inside a `delayed_work`. +unsafe impl HasDelayedWork for Device +where + T: drm::Driver, + T::Data: HasDelayedWork, + C: DeviceContext, +{ +} diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs new file mode 100644 index 000000000..74f6ed690 --- /dev/null +++ b/rust/kernel/drm/driver.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM driver core. +//! +//! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) + +use crate::{ + bindings, + device, + drm, + error::to_result, + prelude::*, + sync::aref::ARef, // +}; +use core::ptr::NonNull; + +/// Driver use the GEM memory manager. This should be set for all modern drivers. +pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports render nodes, i.e.: /dev/dri/renderDXX devices. +pub(crate) const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; + +/// Information data for a DRM Driver. +pub struct DriverInfo { + /// Driver major version. + pub major: i32, + /// Driver minor version. + pub minor: i32, + /// Driver patchlevel version. + pub patchlevel: i32, + /// Driver name. + pub name: &'static CStr, + /// Driver description. + pub desc: &'static CStr, +} + +/// Internal memory management operation set, normally created by memory managers (e.g. GEM). +pub struct AllocOps { + pub(crate) gem_create_object: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) prime_handle_to_fd: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + handle: u32, + flags: u32, + prime_fd: *mut core::ffi::c_int, + ) -> core::ffi::c_int, + >, + pub(crate) prime_fd_to_handle: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + prime_fd: core::ffi::c_int, + handle: *mut u32, + ) -> core::ffi::c_int, + >, + pub(crate) gem_prime_import: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + dma_buf: *mut bindings::dma_buf, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) gem_prime_import_sg_table: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + attach: *mut bindings::dma_buf_attachment, + sgt: *mut bindings::sg_table, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) dumb_create: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + args: *mut bindings::drm_mode_create_dumb, + ) -> core::ffi::c_int, + >, + pub(crate) dumb_map_offset: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + handle: u32, + offset: *mut u64, + ) -> core::ffi::c_int, + >, +} + +/// Trait for memory manager implementations. Implemented internally. +pub trait AllocImpl: super::private::Sealed + drm::gem::IntoGEMObject { + /// The [`Driver`] implementation for this [`AllocImpl`]. + type Driver: drm::Driver; + + /// The C callback operations for this memory manager. + const ALLOC_OPS: AllocOps; +} + +/// The DRM `Driver` trait. +/// +/// This trait must be implemented by drivers in order to create a `struct drm_device` and `struct +/// drm_driver` to be registered in the DRM subsystem. +#[vtable] +pub trait Driver { + /// Context data associated with the DRM driver + type Data: Sync + Send; + + /// Data owned by the [`Registration`] and accessible within a + /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via + /// [`Device::registration_data_with()`](drm::Device::registration_data_with). + /// + /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the + /// parent bus device binding scope but may be shorter. + type RegistrationData<'a>: Send + Sync + 'a; + + /// The type used to manage memory for this driver. + type Object: AllocImpl; + + /// The type used to represent a DRM File (client) + type File: drm::file::DriverFile; + + /// The bus device type of the parent device that the DRM device is associated with. + type ParentDevice: device::AsBusDevice; + + /// Driver metadata + const INFO: DriverInfo; + + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; + + /// Sets the `DRIVER_RENDER` feature for this driver. + /// + /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to + /// userspace. The render node is an alternate low-privilege way to access + /// the driver, which is enforced on a per-ioctl level. Userspace processes + /// that open the render node can only invoke ioctls explicitly listed as + /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas + /// userspace processes using the master node can invoke any ioctl. + const FEAT_RENDER: bool = false; +} + +/// The registration type of a `drm::Device`. +/// +/// Once the `Registration` structure is dropped, the device is unregistered. +pub struct Registration<'a, T: Driver> { + drm: ARef>, + _reg_data: Pin>>, +} + +impl<'a, T: Driver> Registration<'a, T> { + /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + pub unsafe fn new( + dev: &'a device::Device, + drm: drm::UnregisteredDevice, + reg_data: impl PinInit, E>, + flags: usize, + ) -> Result + where + Error: From, + { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { + return Err(EINVAL); + } + + let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once ioctls can be called. + let ptr: NonNull> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + if let Err(e) = to_result(ret) { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent + // access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + return Err(e); + } + + Ok(Self { + drm: (&*drm).into(), + _reg_data: reg_data, + }) + } + + /// Returns a reference to the `Device` instance for this registration. + pub fn device(&self) -> &drm::Device { + &self.drm + } +} + +// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between +// threads, hence it's safe to share it. +unsafe impl Sync for Registration<'_, T> {} + +// SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. +unsafe impl Send for Registration<'_, T> {} + +impl Drop for Registration<'_, T> { + fn drop(&mut self) { + // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing + // `drm_dev_enter()` critical sections complete before unregistration proceeds. This + // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in + // `drm_dev_unplug()` to guarantee that the parent device is still bound within the + // critical section. + // + // SAFETY: Safe by the invariant of `ARef>`. The existence of this + // `Registration` also guarantees that this `drm::Device` is actually registered. + unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) }; + // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical + // sections have completed, so no one holds a reference to reg_data anymore. + // reg_data is dropped here automatically. + } +} diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs new file mode 100644 index 000000000..10160601c --- /dev/null +++ b/rust/kernel/drm/file.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM File objects. +//! +//! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) + +use crate::{ + bindings, + drm, + prelude::*, + types::Opaque, // +}; +use core::marker::PhantomData; + +/// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). +pub trait DriverFile { + /// The parent `Driver` implementation for this `DriverFile`. + type Driver: drm::Driver; + + /// Open a new file (called when a client opens the DRM device). + fn open(device: &drm::Device) -> Result>>; +} + +/// An open DRM File. +/// +/// # Invariants +/// +/// `self.0` is a valid instance of a `struct drm_file`. +#[repr(transparent)] +pub struct File(Opaque, PhantomData); + +impl File { + #[doc(hidden)] + /// Not intended to be called externally, except via declare_drm_ioctls!() + /// + /// # Safety + /// + /// `raw_file` must be a valid pointer to an open `struct drm_file`, opened through `T::open`. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_file) -> &'a File { + // SAFETY: `raw_file` is valid by the safety requirements of this function. + unsafe { &*ptr.cast() } + } + + pub(super) fn as_raw(&self) -> *mut bindings::drm_file { + self.0.get() + } + + fn driver_priv(&self) -> *mut T { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. + unsafe { (*self.as_raw()).driver_priv }.cast() + } + + /// Return a pinned reference to the driver file structure. + pub fn inner(&self) -> Pin<&T> { + // SAFETY: By the type invariant the pointer `self.as_raw()` points to a valid and opened + // `struct drm_file`, hence `driver_priv` has been properly initialized by `open_callback`. + unsafe { Pin::new_unchecked(&*(self.driver_priv())) } + } + + /// The open callback of a `struct drm_file`. + pub(crate) extern "C" fn open_callback( + raw_dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, + ) -> core::ffi::c_int { + // SAFETY: A callback from `struct drm_driver::open` guarantees that + // - `raw_dev` is valid pointer to a `struct drm_device`, + // - the corresponding `struct drm_device` has been registered. + let drm = unsafe { drm::Device::from_raw(raw_dev) }; + + // SAFETY: `raw_file` is a valid pointer to a `struct drm_file`. + let file = unsafe { File::::from_raw(raw_file) }; + + let inner = match T::open(drm) { + Err(e) => { + return e.to_errno(); + } + Ok(i) => i, + }; + + // SAFETY: This pointer is treated as pinned, and the Drop guarantee is upheld in + // `postclose_callback()`. + let driver_priv = KBox::into_raw(unsafe { Pin::into_inner_unchecked(inner) }); + + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. + unsafe { (*file.as_raw()).driver_priv = driver_priv.cast() }; + + 0 + } + + /// The postclose callback of a `struct drm_file`. + pub(crate) extern "C" fn postclose_callback( + _raw_dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: This reference won't escape this function + let file = unsafe { File::::from_raw(raw_file) }; + + // SAFETY: `file.driver_priv` has been created in `open_callback` through `KBox::into_raw`. + let _ = unsafe { KBox::from_raw(file.driver_priv()) }; + } +} + +impl super::private::Sealed for File {} diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs new file mode 100644 index 000000000..e1ebad77e --- /dev/null +++ b/rust/kernel/drm/gem/mod.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM GEM API +//! +//! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) + +use crate::{ + bindings, + drm::{ + self, + device::{ + DeviceContext, + Normal, // + }, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::NonNull, // +}; + +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; + +/// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its +/// [`DriverObject`] implementation. +/// +/// [`Driver`]: drm::Driver +/// [`DriverFile`]: drm::file::DriverFile +pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; + +/// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. +/// +/// [`Driver`]: drm::Driver +pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; + +/// GEM object functions, which must be implemented by drivers. +pub trait DriverObject: Sync + Send + Sized + 'static { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + + /// Create a new driver data object for a GEM object of a given size. + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; + + /// Open a new handle to an existing object, associated with a File. + fn open(_obj: &DriverAllocImpl, _file: &DriverFile) -> Result { + Ok(()) + } + + /// Close a handle to an existing object, associated with a File. + fn close(_obj: &DriverAllocImpl, _file: &DriverFile) {} +} + +/// Trait that represents a GEM object subtype +pub trait IntoGEMObject: Sized + super::private::Sealed { + /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as + /// this owning object is valid. + fn as_raw(&self) -> *mut bindings::drm_gem_object; + + /// Converts a pointer to a `struct drm_gem_object` into a reference to `Self`. + /// + /// # Safety + /// + /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a + /// valid instance of `Self`. + /// - The caller promises that holding the immutable reference returned by this function does + /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; +} + +extern "C" fn open_callback( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) -> core::ffi::c_int { + // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. + let file = unsafe { DriverFile::::from_raw(raw_file) }; + + // SAFETY: + // * `open_callback` is specified in the AllocOps structure for `DriverObject`, ensuring that + // `raw_obj` is contained within a `DriverAllocImpl` + // * It is only possible for `open_callback` to be called after device registration, ensuring + // that the object's device is in the `Registered` state. + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; + + match T::open(obj, file) { + Err(e) => e.to_errno(), + Ok(()) => 0, + } +} + +extern "C" fn close_callback( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) { + // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. + let file = unsafe { DriverFile::::from_raw(raw_file) }; + + // SAFETY: `close_callback` is specified in the AllocOps structure for `Object`, ensuring + // that `raw_obj` is indeed contained within a `Object`. + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; + + T::close(obj, file); +} + +impl IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { + // SAFETY: `obj` is guaranteed to be in an `Object` via the safety contract of this + // function + unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object, obj) } + } +} + +/// Base operations shared by all GEM object classes +pub trait BaseObject: IntoGEMObject { + /// Returns the size of the object in bytes. + fn size(&self) -> usize { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `struct drm_gem_object`. + unsafe { (*self.as_raw()).size } + } + + /// Creates a new handle for the object associated with a given `File` + /// (or returns an existing one). + fn create_handle(&self, file: &drm::File) -> Result + where + Self: AllocImpl, + D: drm::Driver, + F: drm::file::DriverFile, + { + let mut handle: u32 = 0; + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_handle_create(file.as_raw().cast(), self.as_raw(), &mut handle) + })?; + Ok(handle) + } + + /// Looks up an object by its handle for a given `File`. + fn lookup_handle(file: &drm::File, handle: u32) -> Result> + where + Self: AllocImpl + AlwaysRefCounted, + D: drm::Driver, + F: drm::file::DriverFile, + { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_gem_object_lookup(file.as_raw().cast(), handle) }; + if ptr.is_null() { + return Err(ENOENT); + } + + // SAFETY: + // - A `drm::Driver` can only have a single `File` implementation. + // - `file` uses the same `drm::Driver` as `Self`. + // - Therefore, we're guaranteed that `ptr` must be a gem object embedded within `Self`. + // - And we check if the pointer is null befoe calling from_raw(), ensuring that `ptr` is a + // valid pointer to an initialized `Self`. + let obj = unsafe { Self::from_raw(ptr) }; + + // SAFETY: + // - We take ownership of the reference of `drm_gem_object_lookup()`. + // - Our `NonNull` comes from an immutable reference, thus ensuring it is a valid pointer to + // `Self`. + Ok(unsafe { ARef::from_raw(obj.into()) }) + } + + /// Creates an mmap offset to map the object from userspace. + fn create_mmap_offset(&self) -> Result { + // SAFETY: The arguments are valid per the type invariant. + to_result(unsafe { bindings::drm_gem_create_mmap_offset(self.as_raw()) })?; + + // SAFETY: The arguments are valid per the type invariant. + Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) + } +} + +impl BaseObject for T {} + +/// Crate-private base operations shared by all GEM object classes. +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + +/// A base GEM object. +/// +/// # Invariants +/// +/// * `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. +#[repr(C)] +#[pin_data] +pub struct Object { + obj: Opaque, + #[pin] + data: T, + _ctx: PhantomData, +} + +impl Object { + const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(open_callback::), + close: Some(close_callback::), + print_info: None, + export: None, + pin: None, + unpin: None, + get_sg_table: None, + vmap: None, + vunmap: None, + mmap: None, + status: None, + vm_ops: core::ptr::null_mut(), + evict: None, + rss: None, + }; + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &drm::Device { + // SAFETY: + // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM + // object lives. + // - The device we used for creating the gem object is passed as &drm::Device to + // Object::::new(), so we know that `T::Driver` is the right generic parameter to use + // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. + unsafe { drm::Device::from_raw((*self.as_raw()).dev) } + } + + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + let ptr: *mut Opaque = obj.cast(); + + // SAFETY: All of our objects are of type `Object`. + let this = unsafe { crate::container_of!(ptr, Self, obj) }; + + // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct + // drm_gem_object`. + unsafe { bindings::drm_gem_object_release(obj) }; + + // SAFETY: All of our objects are allocated via `KBox`, and we're in the + // free callback which guarantees this object has zero remaining references, + // so we can drop it. + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl Object { + /// Create a new GEM object. + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { + let obj: Pin> = KBox::pin_init( + try_pin_init!(Self { + obj: Opaque::new(bindings::drm_gem_object::default()), + data <- T::new(dev, size, args), + _ctx: PhantomData, + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; + + // INVARIANT: `dev` and the GEM object are in the same state at the moment, and upgrading + // the typestate in `dev` will not carry over to the GEM object. + if let Err(err) = + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) + }) + { + // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before + // failing, so `drm_gem_private_object_fini()` is the matching cleanup. + unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) }; + return Err(err); + } + + // SAFETY: We will never move out of `Self` as `ARef` is always treated as pinned. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); + + // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. + Ok(unsafe { ARef::from_raw(ptr) }) + } +} + +impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject +} + +impl super::private::Sealed for Object {} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: AllocOps = AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: None, + dumb_create: None, + dumb_map_offset: None, + }; +} + +pub(super) const fn create_fops(owner: *mut bindings::module) -> bindings::file_operations { + let mut fops: bindings::file_operations = pin_init::zeroed(); + + fops.owner = owner; + fops.open = Some(bindings::drm_open); + fops.release = Some(bindings::drm_release); + fops.unlocked_ioctl = Some(bindings::drm_ioctl); + #[cfg(CONFIG_COMPAT)] + { + fops.compat_ioctl = Some(bindings::drm_compat_ioctl); + } + fops.poll = Some(bindings::drm_poll); + fops.read = Some(bindings::drm_read); + fops.llseek = Some(bindings::noop_llseek); + fops.mmap = Some(bindings::drm_gem_mmap); + fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET; + + fops +} diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 000000000..a687d46d1 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + device::{ + self, + Bound, // + }, + devres::*, + drm::{ + driver, + gem, + private::Sealed, + Device, // + }, + error::{ + from_err_ptr, + to_result, // + }, + io::{ + IoBase, + Region, + SysMem, + SysMemBackend, // + }, + prelude::*, + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, +}; +use core::{ + ffi::c_void, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::{ + self, + NonNull, // + }, +}; +use gem::{ + BaseObject, + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject +} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Send for Object {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within an + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> + where + R: Deref + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap(&self) -> Result> { + self.make_vmap() + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { + let parent = self.dev().as_ref(); + if dev.as_raw() != parent.as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap(&self) -> Result> { + self.make_vmap() + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Self, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject>(&'a Object, NotThreadSafe); + +impl<'a, T: DriverObject> DmaResvGuard<'a, T> { + #[inline] + fn new(obj: &'a Object) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap +where + D: DriverObject, + R: Deref>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned = VMap>, SIZE>; + +impl VMap +where + D: DriverObject, + R: Deref>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl<'a, D, R, const SIZE: usize> IoBase<'a> for &'a VMap +where + D: DriverObject, + R: Deref>, +{ + type Backend = SysMemBackend; + type Target = Region; + + #[inline] + fn as_view(self) -> SysMem<'a, Region> { + let ptr = Region::ptr_from_raw_parts_mut(self.addr.cast(), self.owner.size()); + + // SAFETY: Per type invariants of `VMap`: + // - `addr .. addr + owner.size()` is a valid kernel accessible memory region. + // - `addr` is page-aligned, which satisfies `Region`'s 4-byte alignment requirement. + // - The memory remains valid until this `VMap` is dropped; since `self` is `&'a VMap`, + // the borrow prevents the `VMap` from being dropped for the lifetime `'a`. + unsafe { SysMem::new(ptr) } + } +} + +impl Drop for VMap +where + D: DriverObject, + R: Deref>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl Send for VMap +where + D: DriverObject, + R: Deref> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl Sync for VMap +where + D: DriverObject, + R: Deref> + Sync, +{ +} + +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, +} + +impl Deref for SGTableMap { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl Drop for SGTableMap { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + io::Io, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device, + _size: usize, + _args: Self::Args, + ) -> impl PinInit { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = KunitFile; + type Object = Object; + type ParentDevice = faux::Device; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let reg = faux::Registration::new(c"Kunit", None)?; + let fdev = reg.as_ref(); + let drm = UnregisteredDevice::new(fdev, data)?; + + Ok((reg, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the size matches the actual object size + assert_eq!(vmap.size(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.try_read8(offset).unwrap(), expected); + } + + Ok(()) + } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"EvilKunit", None)?; + let wrong_dev = reg.as_ref(); + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } +} diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs new file mode 100644 index 000000000..d9d43d719 --- /dev/null +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![cfg(CONFIG_RUST_DRM_GPUVM)] + +//! DRM GPUVM in immediate mode +//! +//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated +//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM +//! and the GPU's virtual address space has the same state at all times. +//! +//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) + +use kernel::{ + alloc::{ + AllocError, + Flags as AllocFlags, // + }, + bindings, + drm, + drm::gem::IntoGEMObject, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Range, // + }, + ptr::{ + self, + NonNull, // + }, // +}; + +mod sm_ops; +pub use self::sm_ops::*; + +mod vm_bo; +pub use self::vm_bo::*; + +mod va; +pub use self::va::*; + +/// A DRM GPU VA manager. +/// +/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed +/// via the special unique handle [`UniqueRefGpuVm`]. +/// +/// # Invariants +/// +/// * Stored in an allocation managed by the refcount in `self.vm`. +/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. +/// * Does not contain any sparse [`GpuVa`] instances. +#[pin_data] +pub struct GpuVm { + #[pin] + vm: Opaque, + /// Accessed only through the [`UniqueRefGpuVm`] reference. + data: UnsafeCell, +} + +// SAFETY: It is safe to send a `GpuVm` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. +unsafe impl Send for GpuVm {} +// SAFETY: It is safe to share a `&GpuVm` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. +unsafe impl Sync for GpuVm {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; + } +} + +impl PartialEq for GpuVm { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVm {} + +impl GpuVm { + const fn vtable() -> &'static bindings::drm_gpuvm_ops { + &bindings::drm_gpuvm_ops { + vm_free: Some(Self::vm_free), + op_alloc: None, + op_free: None, + vm_bo_alloc: GpuVmBo::::ALLOC_FN, + vm_bo_free: GpuVmBo::::FREE_FN, + vm_bo_validate: None, + sm_step_map: Some(Self::sm_step_map), + sm_step_unmap: Some(Self::sm_step_unmap), + sm_step_remap: Some(Self::sm_step_remap), + } + } + + /// Creates a GPUVM instance. + #[expect(clippy::new_ret_no_self)] + pub fn new( + name: &'static CStr, + dev: &drm::Device, + r_obj: &T::Object, + range: Range, + reserve_range: Range, + data: T, + ) -> Result, E> + where + E: From, + E: From, + { + let obj = KBox::try_pin_init::( + try_pin_init!(Self { + data: UnsafeCell::new(data), + vm <- Opaque::ffi_init(|vm| { + // SAFETY: These arguments are valid. `vm` is valid until refcount drops to + // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag + // below. + unsafe { + bindings::drm_gpuvm_init( + vm, + name.as_char_ptr(), + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, + dev.as_raw(), + r_obj.as_raw(), + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + const { Self::vtable() }, + ) + } + }), + }? E), + GFP_KERNEL | __GFP_ZERO, + )?; + // SAFETY: This transfers the initial refcount to the ARef. + let aref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( + Pin::into_inner_unchecked(obj), + ))) + }; + // INVARIANT: This reference is unique. + Ok(UniqueRefGpuVm(aref)) + } + + /// Access this [`GpuVm`] from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm`] that remains + /// valid for at least `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. Caller ensures the + // pointer is valid for 'a. + unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } + } + + /// Returns a raw pointer to the embedded `struct drm_gpuvm`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { + self.vm.get() + } + + /// The start of the VA space. + #[inline] + pub fn va_start(&self) -> u64 { + // SAFETY: The `mm_start` field is immutable. + unsafe { (*self.as_raw()).mm_start } + } + + /// The length of the GPU's virtual address space. + #[inline] + pub fn va_length(&self) -> u64 { + // SAFETY: The `mm_range` field is immutable. + unsafe { (*self.as_raw()).mm_range } + } + + /// Returns the range of the GPU virtual address space. + #[inline] + pub fn va_range(&self) -> Range { + let start = self.va_start(); + // OVERFLOW: This reconstructs the Range passed to the constructor, so it won't fail. + let end = start + self.va_length(); + Range { start, end } + } + + /// Get or create the [`GpuVmBo`] for this gem object. + #[inline] + pub fn obtain( + &self, + obj: &T::Object, + data: impl PinInit, + ) -> Result>, AllocError> { + Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) + } + + /// Clean up buffer objects that are no longer used. + #[inline] + pub fn deferred_cleanup(&self) { + // SAFETY: This GPUVM uses immediate mode. + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } + } + + /// Check if this GEM object is an external object for this GPUVM. + #[inline] + pub fn is_extobj(&self, obj: &T::Object) -> bool { + // SAFETY: We may call this with any GPUVM and GEM object. + unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } + } + + /// Free this GPUVM. + /// + /// # Safety + /// + /// Called when refcount hits zero. + unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. + let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; + // SAFETY: By type invariants we can free it when refcount hits zero. + drop(unsafe { KBox::from_raw(me) }) + } + + #[inline] + fn raw_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. + unsafe { (*(*self.as_raw()).r_obj).resv } + } +} + +/// The manager for a GPUVM. +pub trait DriverGpuVm: Sized + Send + Sync { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The kind of GEM object stored in this GPUVM. + type Object: drm::driver::AllocImpl + Send + Sync; + + /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; + + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). + type VmBoData: Send + Sync; + + /// The private data passed to callbacks. + type SmContext<'ctx> + where + Self: 'ctx; + + /// Indicates that a new mapping should be created. + fn sm_step_map<'op, 'ctx>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op, 'ctx>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + + /// Indicates that an existing mapping should be split up. + fn sm_step_remap<'op, 'ctx>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; +} + +/// The core of the DRM GPU VA manager. +/// +/// This object is a unique reference to the VM that can access the interval tree and the Rust +/// `data` field. +/// +/// # Invariants +/// +/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. +pub struct UniqueRefGpuVm(ARef>); + +impl UniqueRefGpuVm { + /// Access the data owned by this `UniqueRefGpuVm` immutably. + #[inline] + pub fn data_ref(&self) -> &T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &*self.0.data.get() } + } + + /// Access the data owned by this `UniqueRefGpuVm` mutably. + #[inline] + pub fn data(&mut self) -> &mut T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &mut *self.0.data.get() } + } +} + +impl Deref for UniqueRefGpuVm { + type Target = GpuVm; + + #[inline] + fn deref(&self) -> &GpuVm { + &self.0 + } +} diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs new file mode 100644 index 000000000..742c151b2 --- /dev/null +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// The actual data that gets threaded through the callbacks. +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { + gpuvm: &'a mut UniqueRefGpuVm, + user_context: &'a mut T::SmContext<'ctx>, +} + +/// Adds an extra field to `SmData` for `sm_map()` callbacks. +/// +/// # Invariants +/// +/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`. +#[repr(C)] +struct SmMapData<'a, 'ctx, T: DriverGpuVm> { + sm_data: SmData<'a, 'ctx, T>, + vm_bo: &'a GpuVmBo, +} + +/// The argument for [`UniqueRefGpuVm::sm_map`]. +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { + /// Address in GPU virtual address space. + pub addr: u64, + /// Length of mapping to create. + pub range: u64, + /// Offset in GEM object. + pub gem_offset: u64, + /// The GEM object to map. + pub vm_bo: &'a GpuVmBo, + /// The user-provided context type. + pub context: &'a mut T::SmContext<'ctx>, +} + +impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> { + fn raw_request(&self) -> bindings::drm_gpuvm_map_req { + bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: self.addr, + range: self.range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: self.gem_offset, + obj: self.vm_bo.obj().as_raw(), + }, + }, + } + } +} + +/// Represents an `sm_step_map` operation that has not yet been completed. +pub struct OpMap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_map, + // Since these abstractions are designed for immediate mode, the VM BO needs to be + // pre-allocated, so we always have it available when we reach this point. + vm_bo: &'op GpuVmBo, + // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not + // coerce to `OpMap<'short, T>`. This ensures that the user can't return + // the wrong `OpMapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpMap<'op, T> { + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.op.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.op.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.op.gem.offset + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) to map. + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw(self.op.gem.obj) } + } + + /// The [`GpuVmBo`] that the new VA will be associated with. + pub fn vm_bo(&self) -> &GpuVmBo { + self.vm_bo + } + + /// Use the pre-allocated VA to carry out this map operation. + pub fn insert(self, va: GpuVaAlloc, va_data: impl PinInit) -> OpMapped<'op, T> { + let va = va.prepare(va_data); + // SAFETY: By the type invariants we may access the interval tree. + unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) }; + + let _gpuva_guard = self.vm_bo().lock_gpuva(); + // SAFETY: The va is prepared for insertion, and we hold the GEM lock. + unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) }; + + OpMapped { + _invariant: self._invariant, + } + } +} + +/// Represents a completed [`OpMap`] operation. +pub struct OpMapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_unmap` operation that has not yet been completed. +pub struct OpUnmap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_unmap, + // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not + // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the + // wrong`OpUnmapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpUnmap<'op, T> { + /// Indicates whether this [`GpuVa`] is physically contiguous with the + /// original mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table + /// mappings for this `drm_gpuva`, adding the missing page table entries + /// only and update the `drm_gpuvm` accordingly. + pub fn keep(&self) -> bool { + self.op.keep + } + + /// The range being unmapped. + pub fn va(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw(self.op.va) } + } + + /// Remove the VA. + pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved) { + // SAFETY: The op references a valid drm_gpuva in the GPUVM. + unsafe { bindings::drm_gpuva_unmap(self.op) }; + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) }; + + // SAFETY: We just removed this va from the `GpuVm`. + let va = unsafe { GpuVaRemoved::from_raw(self.op.va) }; + + ( + OpUnmapped { + _invariant: self._invariant, + }, + va, + ) + } +} + +/// Represents a completed [`OpUnmap`] operation. +pub struct OpUnmapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_remap` operation that has not yet been completed. +pub struct OpRemap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_remap, + // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not + // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the + // wrong`OpRemapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpRemap<'op, T> { + /// The preceding part of a split mapping. + #[inline] + pub fn prev(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// The subsequent part of a split mapping. + #[inline] + pub fn next(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original + /// mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this + /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm` + /// accordingly. + #[inline] + pub fn keep(&self) -> bool { + // SAFETY: The unmap pointer is always valid. + unsafe { (*self.op.unmap).keep } + } + + /// The range being unmapped. + #[inline] + pub fn va_to_unmap(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw((*self.op.unmap).va) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped. + #[inline] + pub fn obj(&self) -> &T::Object { + self.va_to_unmap().obj() + } + + /// The [`GpuVmBo`] that is being remapped. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + self.va_to_unmap().vm_bo() + } + + /// Update the GPUVM to perform the remapping. + pub fn remap( + self, + va_alloc: [GpuVaAlloc; 2], + prev_data: impl PinInit, + next_data: impl PinInit, + ) -> (OpRemapped<'op, T>, OpRemapRet) { + let [va1, va2] = va_alloc; + + let mut unused_va = None; + let mut prev_ptr = ptr::null_mut(); + let mut next_ptr = ptr::null_mut(); + if self.prev().is_some() { + prev_ptr = va1.prepare(prev_data); + } else { + unused_va = Some(va1); + } + if self.next().is_some() { + next_ptr = va2.prepare(next_data); + } else { + unused_va = Some(va2); + } + + // SAFETY: the pointers are non-null when required + unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) }; + + let gpuva_guard = self.vm_bo().lock_gpuva(); + if !prev_ptr.is_null() { + // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) }; + } + if !next_ptr.is_null() { + // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) }; + } + drop(gpuva_guard); + + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) }; + + ( + OpRemapped { + _invariant: self._invariant, + }, + OpRemapRet { + // SAFETY: We just removed this va from the `GpuVm`. + unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) }, + unused_va, + }, + ) + } +} + +/// Part of an [`OpRemap`] that represents a new mapping. +#[repr(transparent)] +pub struct OpRemapMapData(bindings::drm_gpuva_op_map); + +impl OpRemapMapData { + /// # Safety + /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`. + unsafe fn from_raw<'a>(ptr: NonNull) -> &'a Self { + // SAFETY: ok per safety requirements + unsafe { ptr.cast().as_ref() } + } + + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.0.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.0.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.0.gem.offset + } +} + +/// Struct containing objects removed or not used by [`OpRemap::remap`]. +pub struct OpRemapRet { + /// The `drm_gpuva` that was removed. + pub unmapped_va: GpuVaRemoved, + /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is + /// returned here. + pub unused_va: Option>, +} + +/// Represents a completed [`OpRemap`] operation. +pub struct OpRemapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +impl UniqueRefGpuVm { + /// Create a mapping, removing or remapping anything that overlaps. + /// + /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that + /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping. + #[inline] + pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result { + if req.vm_bo.gpuvm() != &**self { + return Err(EINVAL); + } + + let gpuvm = self.as_raw(); + let raw_req = req.raw_request(); + // INVARIANT: Checked above that `vm_bo.gpuvm() == self`. + let mut p = SmMapData { + sm_data: SmData { + gpuvm: self, + user_context: req.context, + }, + vm_bo: req.vm_bo, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is valid to be interpreted as both SmData and SmMapData since the + // first field of SmMapData is SmData. + to_result(unsafe { + bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req) + }) + } + + /// Remove any mappings in the given region. + /// + /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the + /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range. + #[inline] + pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result { + let gpuvm = self.as_raw(); + let mut p = SmData { + gpuvm: self, + user_context: context, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is a valid SmData. + to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) + } +} + +impl GpuVm { + /// # Safety + /// Must be called from `sm_map` with a pointer to `SmMapData`. + pub(super) unsafe extern "C" fn sm_step_map( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes + // an `SmMapData` as private data. + let p = unsafe { &mut *p.cast::>() }; + let op = OpMap { + // SAFETY: sm_step_map is called with a map operation. + op: unsafe { &(*op).__bindgen_anon_1.map }, + vm_bo: p.vm_bo, + _invariant: PhantomData, + }; + match p + .sm_data + .gpuvm + .data() + .sm_step_map(op, p.sm_data.user_context) + { + Ok(OpMapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_unmap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpUnmap { + // SAFETY: sm_step_unmap is called with an unmap operation. + op: unsafe { &(*op).__bindgen_anon_1.unmap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_unmap(op, p.user_context) { + Ok(OpUnmapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_remap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpRemap { + // SAFETY: sm_step_remap is called with a remap operation. + op: unsafe { &(*op).__bindgen_anon_1.remap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_remap(op, p.user_context) { + Ok(OpRemapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } +} diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs new file mode 100644 index 000000000..46493f0ec --- /dev/null +++ b/rust/kernel/drm/gpuvm/va.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * This is a valid `drm_gpuva` object that is resident in a [`GpuVm`] instance. +/// * It is associated with a [`GpuVmBo`]. Or in other words, it's not an +/// `gpuvm->kernel_alloc_node` and `DRM_GPUVA_SPARSE` is not set. +/// * The associated [`GpuVmBo`] is part of the GEM list. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + inner: Opaque, + #[pin] + data: T::VaData, +} + +impl PartialEq for GpuVa { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVa {} + +impl GpuVa { + /// Access this [`GpuVa`] from a raw pointer. + /// + /// # Safety + /// + /// * For the duration of `'a`, the pointer must reference a valid `drm_gpuva` associated with + /// a [`GpuVm`]. + /// * It must be associated with a [`GpuVmBo`]. + /// * The associated [`GpuVmBo`] is part of the GEM list. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuva) -> &'a Self { + // CAST: `drm_gpuva` is first field and `repr(C)`. + // SAFETY: The safety requirements match the invariants of `GpuVa`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuva { + self.inner.get() + } + + /// Returns the address of this mapping in the GPU virtual address space. + #[inline] + pub fn addr(&self) -> u64 { + // SAFETY: The `va.addr` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.addr } + } + + /// Returns the length of this mapping. + #[inline] + pub fn length(&self) -> u64 { + // SAFETY: The `va.range` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.range } + } + + /// Returns `addr..addr+length`. + #[inline] + pub fn range(&self) -> Range { + let addr = self.addr(); + addr..addr + self.length() + } + + /// Returns the offset within the GEM object. + #[inline] + pub fn gem_offset(&self) -> u64 { + // SAFETY: The `gem.offset` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).gem.offset } + } + + /// Returns the GEM object. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `gem.obj` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. + unsafe { ::from_raw((*self.as_raw()).gem.obj) } + } + + /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`]. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + // SAFETY: The `vm_bo` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. The BO is in the GEM list by the type + // invariants. + unsafe { GpuVmBo::from_raw((*self.as_raw()).vm_bo) } + } +} + +/// A pre-allocated [`GpuVa`] object. +/// +/// # Invariants +/// +/// The memory is zeroed. +pub struct GpuVaAlloc(KBox>>); + +// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no +// thread-bound state. +unsafe impl Send for GpuVaAlloc {} + +// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared +// `&GpuVaAlloc` cannot access the allocation. +unsafe impl Sync for GpuVaAlloc {} + +impl GpuVaAlloc { + /// Pre-allocate a [`GpuVa`] object. + pub fn new(flags: AllocFlags) -> Result, AllocError> { + // INVARIANTS: Memory allocated with __GFP_ZERO. + Ok(GpuVaAlloc(KBox::new_uninit(flags | __GFP_ZERO)?)) + } + + /// Prepare this `drm_gpuva` for insertion into the GPUVM. + #[must_use] + pub(super) fn prepare(mut self, va_data: impl PinInit) -> *mut bindings::drm_gpuva { + let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); + // SAFETY: The `data` field is pinned. + unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) }; + KBox::into_raw(self.0).cast() + } +} + +/// A [`GpuVa`] object that has been removed. +/// +/// # Invariants +/// +/// The `drm_gpuva` is not resident in the [`GpuVm`]. +pub struct GpuVaRemoved(KBox>); + +impl GpuVaRemoved { + /// Convert a raw pointer into a [`GpuVaRemoved`]. + /// + /// # Safety + /// + /// * Must have been removed from a [`GpuVm`]. + /// * It must not be a `gpuvm->kernel_alloc_node` va. + pub(super) unsafe fn from_raw(ptr: *mut bindings::drm_gpuva) -> Self { + // SAFETY: Since it used to be a VA in a `GpuVm` and it's not a kernel_alloc_node, this + // pointer references a `GpuVa` with a valid `T::VaData`. Since it has been removed, we + // can take ownership of the allocation. + GpuVaRemoved(unsafe { KBox::from_raw(ptr.cast()) }) + } + + /// Take ownership of the VA data. + pub fn into_inner(self) -> T::VaData + where + T::VaData: Unpin, + { + KBox::into_inner(self.0).data + } +} + +impl Deref for GpuVaRemoved { + type Target = T::VaData; + fn deref(&self) -> &T::VaData { + &self.0.data + } +} + +impl DerefMut for GpuVaRemoved +where + T::VaData: Unpin, +{ + fn deref_mut(&mut self) -> &mut T::VaData { + &mut self.0.data + } +} diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs new file mode 100644 index 000000000..5989972da --- /dev/null +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * Allocated with `kmalloc` and refcounted via `inner`. +/// * Is present in the gem list. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + inner: Opaque, + #[pin] + data: T::VmBoData, +} + +// SAFETY: It is safe to send a `GpuVmBo` to another thread: dropping it there drops +// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds. +unsafe impl Send for GpuVmBo {} + +// SAFETY: It is safe to share a `&GpuVmBo` between threads: it effectively shares +// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an +// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds. +unsafe impl Sync for GpuVmBo {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // CAST: `drm_gpuvm_bo` is first field of repr(C) struct. + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred + // mechanism. + unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) }; + } +} + +impl PartialEq for GpuVmBo { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVmBo {} + +impl GpuVmBo { + /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable. + /// + /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to + /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`. + /// + /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa. + pub(super) const ALLOC_FN: Option *mut bindings::drm_gpuvm_bo> = { + use core::alloc::Layout; + let base = Layout::new::(); + let rust = Layout::new::(); + assert!(base.size() <= rust.size()); + if base.size() != rust.size() || base.align() != rust.align() { + Some(Self::vm_bo_alloc) + } else { + // This causes GPUVM to allocate a `GpuVmBo` with `kzalloc(sizeof(drm_gpuvm_bo))`. + None + } + }; + + /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable. + /// + /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if + /// the default gpuvm behavior is the same as `vm_bo_free`. + /// + /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa. + pub(super) const FREE_FN: Option = { + if core::mem::needs_drop::() { + Some(Self::vm_bo_free) + } else { + // This causes GPUVM to free a `GpuVmBo` with `kfree`. + None + } + }; + + /// Custom function for allocating a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// Always safe to call. + unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo { + let raw_ptr = KBox::::new_uninit(GFP_KERNEL | __GFP_ZERO) + .map(KBox::into_raw) + .unwrap_or(ptr::null_mut()); + + // CAST: `drm_gpuvm_bo` is first field of `Self`. + raw_ptr.cast() + } + + /// Custom function for freeing a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after + /// this call. + unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) { + // CAST: `drm_gpuvm_bo` is first field of `Self`. + // SAFETY: + // * The ptr was allocated from kmalloc with the layout of `GpuVmBo`. + // * `ptr->inner` has no destructor. + // * `ptr->data` contains a valid `T::VmBoData` that we can drop. + drop(unsafe { KBox::::from_raw(ptr.cast()) }); + } + + /// Access this [`GpuVmBo`] from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with + /// a [`GpuVm`]. The BO must also be present in the GEM list. + #[inline] + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self { + // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + self.inner.get() + } + + /// The [`GpuVm`] that this GEM object is mapped in. + #[inline] + pub fn gpuvm(&self) -> &GpuVm { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { GpuVm::::from_raw((*self.inner.get()).vm) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw((*self.inner.get()).obj) } + } + + /// The driver data with this buffer object. + #[inline] + pub fn data(&self) -> &T::VmBoData { + &self.data + } + + pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> { + // SAFETY: The GEM object is valid. + let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock }; + // SAFETY: The GEM object is valid, so the mutex is properly initialized. + let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) }; + mutex.lock() + } +} + +/// A pre-allocated [`GpuVmBo`] object. +/// +/// # Invariants +/// +/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is +/// absent from any gem, extobj, or evict lists. +pub(super) struct GpuVmBoAlloc(NonNull>); + +impl GpuVmBoAlloc { + /// Create a new pre-allocated [`GpuVmBo`]. + /// + /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call + /// drop on the data, so we don't have a way to free it when the data is missing. + #[inline] + pub(super) fn new( + gpuvm: &GpuVm, + gem: &T::Object, + value: impl PinInit, + ) -> Result, AllocError> { + // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout + // of `GpuVmBo`. The type is repr(C), so `container_of` is not required. + // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call. + let raw_ptr = unsafe { + bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::>() + }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + // SAFETY: `ptr->data` is a valid pinned location. + unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) }; + // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid + // as we just initialized it. + Ok(GpuVmBoAlloc(ptr)) + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + // SAFETY: The pointer references a valid `drm_gpuvm_bo`. + unsafe { (*self.0.as_ptr()).inner.get() } + } + + /// Look up whether there is an existing [`GpuVmBo`] for this gem object. + /// + /// The caller should not hold the GEM mutex or DMA resv lock. + #[inline] + pub(super) fn obtain(self) -> ARef> { + let me = ManuallyDrop::new(self); + // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this + // call. + let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) }; + + // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr + let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list. + // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`. + let ret = unsafe { ARef::>::from_raw(nonnull) }; + + // Ensure that external objects are in the extobj list. + // + // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where + // we could end up using the extobj before the thread with `ptr == me` calls extobj_add. + if ret.gpuvm().is_extobj(ret.obj()) { + let resv_lock = ret.gpuvm().raw_resv(); + // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists. + // SAFETY: The GPUVM is still alive, so its resv lock is too. + unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) }; + // SAFETY: We hold the GPUVMs resv lock. + unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) }; + // SAFETY: We took the lock, so we can unlock it. + unsafe { bindings::dma_resv_unlock(resv_lock) }; + } + + ret + } +} + +impl Deref for GpuVmBoAlloc { + type Target = GpuVmBo; + #[inline] + fn deref(&self) -> &GpuVmBo { + // SAFETY: By the type invariants we may deref while `Self` exists. + unsafe { self.0.as_ref() } + } +} + +impl Drop for GpuVmBoAlloc { + #[inline] + fn drop(&mut self) { + // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly. + // SAFETY: It's safe to perform a deferred put in any context. + unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) }; + } +} diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs new file mode 100644 index 000000000..64af9eacc --- /dev/null +++ b/rust/kernel/drm/ioctl.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM IOCTL definitions. +//! +//! C header: [`include/drm/drm_ioctl.h`](srctree/include/drm/drm_ioctl.h) + +use crate::ioctl; + +const BASE: u32 = uapi::DRM_IOCTL_BASE as u32; + +/// Construct a DRM ioctl number with no argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IO(nr: u32) -> u32 { + ioctl::_IO(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-only argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOR(nr: u32) -> u32 { + ioctl::_IOR::(BASE, nr) +} + +/// Construct a DRM ioctl number with a write-only argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOW(nr: u32) -> u32 { + ioctl::_IOW::(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-write argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOWR(nr: u32) -> u32 { + ioctl::_IOWR::(BASE, nr) +} + +/// Descriptor type for DRM ioctls. Use the `declare_drm_ioctls!{}` macro to construct them. +pub type DrmIoctlDescriptor = bindings::drm_ioctl_desc; + +/// This is for ioctl which are used for rendering, and require that the file descriptor is either +/// for a render node, or if it’s a legacy/primary node, then it must be authenticated. +pub const AUTH: u32 = bindings::drm_ioctl_flags_DRM_AUTH; + +/// This must be set for any ioctl which can change the modeset or display state. Userspace must +/// call the ioctl through a primary node, while it is the active master. +/// +/// Note that read-only modeset ioctl can also be called by unauthenticated clients, or when a +/// master is not the currently active one. +pub const MASTER: u32 = bindings::drm_ioctl_flags_DRM_MASTER; + +/// Anything that could potentially wreak a master file descriptor needs to have this flag set. +/// +/// Current that’s only for the SETMASTER and DROPMASTER ioctl, which e.g. logind can call to +/// force a non-behaving master (display compositor) into compliance. +/// +/// This is equivalent to callers with the SYSADMIN capability. +pub const ROOT_ONLY: u32 = bindings::drm_ioctl_flags_DRM_ROOT_ONLY; + +/// This is used for all ioctl needed for rendering only, for drivers which support render nodes. +/// This should be all new render drivers, and hence it should be always set for any ioctl with +/// `AUTH` set. Note though that read-only query ioctl might have this set, but have not set +/// DRM_AUTH because they do not require authentication. +pub const RENDER_ALLOW: u32 = bindings::drm_ioctl_flags_DRM_RENDER_ALLOW; + +/// Internal structures used by the `declare_drm_ioctls!{}` macro. Do not use directly. +#[doc(hidden)] +pub mod internal { + pub use bindings::drm_device; + pub use bindings::drm_file; + pub use bindings::drm_ioctl_desc; + + /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type + /// parameter `T`. + /// + /// Used by [`declare_drm_ioctls!`] to anchor type inference. + #[doc(hidden)] + #[inline] + pub const fn __dev_ctx_cast( + ptr: *const crate::drm::Device, + ) -> *const crate::drm::Device { + ptr.cast() + } +} + +/// Declare the DRM ioctls for a driver. +/// +/// Each entry in the list should have the form: +/// +/// `(ioctl_number, argument_type, flags, user_callback),` +/// +/// `argument_type` is the type name within the `bindings` crate. +/// `user_callback` should have the following prototype: +/// +/// ```ignore +/// fn foo(device: &kernel::drm::Device, +/// reg_data: &Self::RegistrationData<'_>, +/// data: &mut uapi::argument_type, +/// file: &kernel::drm::File, +/// ) -> Result +/// ``` +/// where `Self` is the drm::drv::Driver implementation these ioctls are being declared within. +/// +/// # Examples +/// +/// ```ignore +/// kernel::declare_drm_ioctls! { +/// (FOO_GET_PARAM, drm_foo_get_param, ioctl::RENDER_ALLOW, my_get_param_handler), +/// } +/// ``` +/// +#[macro_export] +macro_rules! declare_drm_ioctls { + ( $(($cmd:ident, $struct:ident, $flags:expr, $func:expr)),* $(,)? ) => { + const IOCTLS: &'static [$crate::drm::ioctl::DrmIoctlDescriptor] = { + use $crate::uapi::*; + const _:() = { + let i: u32 = $crate::uapi::DRM_COMMAND_BASE; + // Assert that all the IOCTLs are in the right order and there are no gaps, + // and that the size of the specified type is correct. + $( + let cmd: u32 = $crate::macros::concat_idents!(DRM_IOCTL_, $cmd); + ::core::assert!(i == $crate::ioctl::_IOC_NR(cmd)); + ::core::assert!(core::mem::size_of::<$crate::uapi::$struct>() == + $crate::ioctl::_IOC_SIZE(cmd)); + let i: u32 = i + 1; + )* + }; + + let ioctls = &[$( + $crate::drm::ioctl::internal::drm_ioctl_desc { + cmd: $crate::macros::concat_idents!(DRM_IOCTL_, $cmd) as u32, + func: { + #[allow(non_snake_case)] + unsafe extern "C" fn $cmd( + raw_dev: *mut $crate::drm::ioctl::internal::drm_device, + raw_data: *mut ::core::ffi::c_void, + raw_file: *mut $crate::drm::ioctl::internal::drm_file, + ) -> core::ffi::c_int { + // SAFETY: + // - The DRM core ensures the device lives while callbacks are being + // called. + // - The DRM device must have been registered when we're called through + // an IOCTL. + // + // INVARIANT: The `Ioctl` context requires that the device has been + // registered via `drm_dev_register()` at some point; the DRM core + // guarantees this for ioctl dispatch callbacks. + // + // FIXME: Currently there is nothing enforcing that the types of the + // dev/file match the current driver these ioctls are being declared + // for, and it's not clear how to enforce this within the type system. + let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> = + $crate::drm::device::Device::from_raw(raw_dev); + + // Type-inference anchor: the closure is never called but ties `dev`'s + // type to `$func`'s first parameter, which the compiler cannot infer + // through method resolution and associated-type projections alone. + #[allow(unreachable_code)] + let _ = || { + let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast( + ::core::ptr::from_ref(dev), + ); + + $func( + // SAFETY: This closure is never executed; the dereference + // exists purely to unify the type parameter with `$func`. + // The pointer is valid regardless. + unsafe { &*__ptr }, + unreachable!(), + unreachable!(), + unreachable!(), + ) + }; + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func; + + let Some(guard) = dev.registration_guard() else { + return $crate::error::code::ENODEV.to_errno(); + }; + + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we + // asserted above matches the size of this type, and all bit patterns of + // UAPI structs must be valid. + // The `ioctl` argument is exclusively owned by the handler + // and guaranteed by the C implementation (`drm_ioctl()`) to remain + // valid for the entire lifetime of the reference taken here. + // There is no concurrent access or aliasing; no other references + // to this object exist during this call. + let data = unsafe { &mut *(raw_data.cast::<$crate::uapi::$struct>()) }; + // SAFETY: This is just the DRM file structure + let file = unsafe { $crate::drm::File::from_raw(raw_file) }; + + match guard.registration_data_with(|reg_data| { + $func(&*guard, reg_data, data, file) + }) { + Err(e) => e.to_errno(), + Ok(i) => i.try_into() + .unwrap_or($crate::error::code::ERANGE.to_errno()), + } + } + Some($cmd) + }, + flags: $flags, + name: $crate::str::as_char_ptr_in_const_context( + $crate::c_str!(::core::stringify!($cmd)), + ), + } + ),*]; + ioctls + }; + }; +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs new file mode 100644 index 000000000..fd6ed35bc --- /dev/null +++ b/rust/kernel/drm/mod.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM subsystem abstractions. + +pub mod device; +pub mod driver; +pub mod file; +pub mod gem; +pub mod gpuvm; +pub mod ioctl; + +pub use self::device::Device; +pub use self::device::DeviceContext; +pub use self::device::Ioctl; +pub use self::device::Normal; +pub use self::device::Registered; +pub use self::device::RegistrationGuard; +pub use self::device::UnregisteredDevice; +pub use self::driver::Driver; +pub use self::driver::DriverInfo; +pub use self::driver::Registration; +pub use self::file::File; + +pub(crate) mod private { + pub trait Sealed {} +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs new file mode 100644 index 000000000..e52793f77 --- /dev/null +++ b/rust/kernel/error.rs @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel errors. +//! +//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)\ +//! C header: [`include/uapi/asm-generic/errno.h`](srctree/include/uapi/asm-generic/errno.h)\ +//! C header: [`include/linux/errno.h`](srctree/include/linux/errno.h) + +use crate::{ + alloc::{layout::LayoutError, AllocError}, + fmt, + str::CStr, +}; + +use core::num::NonZeroI32; +use core::num::TryFromIntError; +use core::str::Utf8Error; + +/// Contains the C-compatible error codes. +#[rustfmt::skip] +pub mod code { + macro_rules! declare_err { + ($err:tt $(,)? $($doc:expr),+) => { + $( + #[doc = $doc] + )* + pub const $err: super::Error = + super::Error::try_from_errno(-(crate::bindings::$err as i32)) + .expect("Invalid errno in `declare_err!`"); + }; + } + + // From `include/uapi/asm-generic/errno-base.h`. + declare_err!(EPERM, "Operation not permitted."); + declare_err!(ENOENT, "No such file or directory."); + declare_err!(ESRCH, "No such process."); + declare_err!(EINTR, "Interrupted system call."); + declare_err!(EIO, "I/O error."); + declare_err!(ENXIO, "No such device or address."); + declare_err!(E2BIG, "Argument list too long."); + declare_err!(ENOEXEC, "Exec format error."); + declare_err!(EBADF, "Bad file number."); + declare_err!(ECHILD, "No child processes."); + declare_err!(EAGAIN, "Try again."); + declare_err!(ENOMEM, "Out of memory."); + declare_err!(EACCES, "Permission denied."); + declare_err!(EFAULT, "Bad address."); + declare_err!(ENOTBLK, "Block device required."); + declare_err!(EBUSY, "Device or resource busy."); + declare_err!(EEXIST, "File exists."); + declare_err!(EXDEV, "Cross-device link."); + declare_err!(ENODEV, "No such device."); + declare_err!(ENOTDIR, "Not a directory."); + declare_err!(EISDIR, "Is a directory."); + declare_err!(EINVAL, "Invalid argument."); + declare_err!(ENFILE, "File table overflow."); + declare_err!(EMFILE, "Too many open files."); + declare_err!(ENOTTY, "Not a typewriter."); + declare_err!(ETXTBSY, "Text file busy."); + declare_err!(EFBIG, "File too large."); + declare_err!(ENOSPC, "No space left on device."); + declare_err!(ESPIPE, "Illegal seek."); + declare_err!(EROFS, "Read-only file system."); + declare_err!(EMLINK, "Too many links."); + declare_err!(EPIPE, "Broken pipe."); + declare_err!(EDOM, "Math argument out of domain of func."); + declare_err!(ERANGE, "Math result not representable."); + + // From `include/uapi/asm-generic/errno.h`. + declare_err!(EDEADLK, "Resource deadlock would occur."); + declare_err!(ENAMETOOLONG, "File name too long."); + declare_err!(ENOLCK, "No record locks available."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EFSBADCRC, "Bad CRC detected."); + declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); + declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); + declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(EFSCORRUPTED, "Filesystem is corrupted."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(EFTYPE, "Wrong file type for the intended operation."); + + // From `include/linux/errno.h`. + declare_err!(ERESTARTSYS, "Restart the system call."); + declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); + declare_err!(ERESTARTNOHAND, "Restart if no handler."); + declare_err!(ENOIOCTLCMD, "No ioctl command."); + declare_err!(ERESTART_RESTARTBLOCK, "Restart by calling sys_restart_syscall."); + declare_err!(EPROBE_DEFER, "Driver requests probe retry."); + declare_err!(EOPENSTALE, "Open found a stale dentry."); + declare_err!(ENOPARAM, "Parameter not supported."); + declare_err!(EBADHANDLE, "Illegal NFS file handle."); + declare_err!(ENOTSYNC, "Update synchronization mismatch."); + declare_err!(EBADCOOKIE, "Cookie is stale."); + declare_err!(ENOTSUPP, "Operation is not supported."); + declare_err!(ETOOSMALL, "Buffer or request is too small."); + declare_err!(ESERVERFAULT, "An untranslatable error occurred."); + declare_err!(EBADTYPE, "Type not supported by server."); + declare_err!(EJUKEBOX, "Request initiated, but will not complete before timeout."); + declare_err!(EIOCBQUEUED, "iocb queued, will get completion event."); + declare_err!(ERECALLCONFLICT, "Conflict with recalled state."); + declare_err!(ENOGRACE, "NFS file lock reclaim refused."); +} + +/// Generic integer kernel error. +/// +/// The kernel defines a set of integer generic error codes based on C and +/// POSIX ones. These codes may have a more specific meaning in some contexts. +/// +/// # Invariants +/// +/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Error(NonZeroI32); + +impl Error { + /// Creates an [`Error`] from a kernel error code. + /// + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + /// + /// It is a bug to pass an out-of-range `errno`. [`code::EINVAL`] is returned in such a case. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Error::from_errno(-1), EPERM); + /// assert_eq!(Error::from_errno(-2), ENOENT); + /// ``` + /// + /// The following calls are considered a bug: + /// + /// ``` + /// assert_eq!(Error::from_errno(0), EINVAL); + /// assert_eq!(Error::from_errno(-1000000), EINVAL); + /// ``` + pub fn from_errno(errno: crate::ffi::c_int) -> Error { + if let Some(error) = Self::try_from_errno(errno) { + error + } else { + // TODO: Make it a `WARN_ONCE` once available. + crate::pr_warn!( + "attempted to create `Error` with out of range `errno`: {}\n", + errno + ); + code::EINVAL + } + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// Returns [`None`] if `errno` is out-of-range. + const fn try_from_errno(errno: crate::ffi::c_int) -> Option { + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + return None; + } + + // SAFETY: `errno` is checked above to be in a valid range. + Some(unsafe { Error::from_errno_unchecked(errno) }) + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// # Safety + /// + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + const unsafe fn from_errno_unchecked(errno: crate::ffi::c_int) -> Error { + // INVARIANT: The contract ensures the type invariant + // will hold. + // SAFETY: The caller guarantees `errno` is non-zero. + Error(unsafe { NonZeroI32::new_unchecked(errno) }) + } + + /// Returns the kernel error code. + pub fn to_errno(self) -> crate::ffi::c_int { + self.0.get() + } + + #[cfg(CONFIG_BLOCK)] + pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { + // SAFETY: `self.0` is a valid error due to its invariant. + unsafe { bindings::errno_to_blk_status(self.0.get()) } + } + + /// Returns the error encoded as a pointer. + pub fn to_ptr(self) -> *mut T { + // SAFETY: `self.0` is a valid error due to its invariant. + unsafe { bindings::ERR_PTR(self.0.get() as crate::ffi::c_long).cast() } + } + + /// Returns a string representing the error, if one exists. + #[cfg(not(testlib))] + pub fn name(&self) -> Option<&'static CStr> { + // SAFETY: Just an FFI call, there are no extra safety requirements. + let ptr = unsafe { bindings::errname(-self.0.get()) }; + if ptr.is_null() { + None + } else { + use crate::str::CStrExt as _; + + // SAFETY: The string returned by `errname` is static and `NUL`-terminated. + Some(unsafe { CStr::from_char_ptr(ptr) }) + } + } + + /// Returns a string representing the error, if one exists. + /// + /// When `testlib` is configured, this always returns `None` to avoid the dependency on a + /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still + /// run in userspace. + #[cfg(testlib)] + pub fn name(&self) -> Option<&'static CStr> { + None + } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.name() { + // Print out number if no name can be found. + None => f.debug_tuple("Error").field(&-self.0).finish(), + Some(name) => f + .debug_tuple( + // SAFETY: These strings are ASCII-only. + unsafe { core::str::from_utf8_unchecked(name.to_bytes()) }, + ) + .finish(), + } + } +} + +impl From for Error { + #[inline] + fn from(_: AllocError) -> Error { + code::ENOMEM + } +} + +impl From for Error { + #[inline] + fn from(_: TryFromIntError) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(_: Utf8Error) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(_: LayoutError) -> Error { + code::ENOMEM + } +} + +impl From for Error { + #[inline] + fn from(_: fmt::Error) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(e: core::convert::Infallible) -> Error { + match e {} + } +} + +/// A [`Result`] with an [`Error`] error type. +/// +/// To be used as the return type for functions that may fail. +/// +/// # Error codes in C and Rust +/// +/// In C, it is common that functions indicate success or failure through +/// their return value; modifying or returning extra data through non-`const` +/// pointer parameters. In particular, in the kernel, functions that may fail +/// typically return an `int` that represents a generic error code. We model +/// those as [`Error`]. +/// +/// In Rust, it is idiomatic to model functions that may fail as returning +/// a [`Result`]. Since in the kernel many functions return an error code, +/// [`Result`] is a type alias for a [`core::result::Result`] that uses +/// [`Error`] as its error type. +/// +/// Note that even if a function does not return anything when it succeeds, +/// it should still be modeled as returning a [`Result`] rather than +/// just an [`Error`]. +/// +/// Calling a function that returns [`Result`] forces the caller to handle +/// the returned [`Result`]. +/// +/// This can be done "manually" by using [`match`]. Using [`match`] to decode +/// the [`Result`] is similar to C where all the return value decoding and the +/// error handling is done explicitly by writing handling code for each +/// error to cover. Using [`match`] the error and success handling can be +/// implemented in all detail as required. For example (inspired by +/// [`samples/rust/rust_minimal.rs`]): +/// +/// ``` +/// # #[allow(clippy::single_match)] +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// match numbers.push(72, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(108, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(200, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// An alternative to be more concise is the [`if let`] syntax: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// if let Err(e) = numbers.push(72, GFP_KERNEL) { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(108, GFP_KERNEL) { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(200, GFP_KERNEL) { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Instead of these verbose [`match`]/[`if let`], the [`?`] operator can +/// be used to handle the [`Result`]. Using the [`?`] operator is often +/// the best choice to handle [`Result`] in a non-verbose way as done in +/// [`samples/rust/rust_minimal.rs`]: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// numbers.push(72, GFP_KERNEL)?; +/// numbers.push(108, GFP_KERNEL)?; +/// numbers.push(200, GFP_KERNEL)?; +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Another possibility is to call [`unwrap()`](Result::unwrap) or +/// [`expect()`](Result::expect). However, use of these functions is +/// *heavily discouraged* in the kernel because they trigger a Rust +/// [`panic!`] if an error happens, which may destabilize the system or +/// entirely break it as a result -- just like the C [`BUG()`] macro. +/// Please see the documentation for the C macro [`BUG()`] for guidance +/// on when to use these functions. +/// +/// Alternatively, depending on the use case, using [`unwrap_or()`], +/// [`unwrap_or_else()`], [`unwrap_or_default()`] or [`unwrap_unchecked()`] +/// might be an option, as well. +/// +/// For even more details, please see the [Rust documentation]. +/// +/// [`match`]: https://doc.rust-lang.org/reference/expressions/match-expr.html +/// [`samples/rust/rust_minimal.rs`]: srctree/samples/rust/rust_minimal.rs +/// [`if let`]: https://doc.rust-lang.org/reference/expressions/if-expr.html#if-let-expressions +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator +/// [`unwrap()`]: Result::unwrap +/// [`expect()`]: Result::expect +/// [`BUG()`]: https://docs.kernel.org/process/deprecated.html#bug-and-bug-on +/// [`unwrap_or()`]: Result::unwrap_or +/// [`unwrap_or_else()`]: Result::unwrap_or_else +/// [`unwrap_or_default()`]: Result::unwrap_or_default +/// [`unwrap_unchecked()`]: Result::unwrap_unchecked +/// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html +pub type Result = core::result::Result; + +/// Converts an integer as returned by a C kernel function to a [`Result`]. +/// +/// If the integer is negative, an [`Err`] with an [`Error`] as given by [`Error::from_errno`] is +/// returned. This means the integer must be `>= -MAX_ERRNO`. +/// +/// Otherwise, it returns [`Ok`]. +/// +/// It is a bug to pass an out-of-range negative integer. `Err(EINVAL)` is returned in such a case. +/// +/// # Examples +/// +/// This function may be used to easily perform early returns with the [`?`] operator when working +/// with C APIs within Rust abstractions: +/// +/// ``` +/// # use kernel::error::to_result; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn f1() -> c_int { 0 } +/// # pub(super) unsafe fn f2() -> c_int { EINVAL.to_errno() } +/// # } +/// fn f() -> Result { +/// // SAFETY: ... +/// to_result(unsafe { bindings::f1() })?; +/// +/// // SAFETY: ... +/// to_result(unsafe { bindings::f2() })?; +/// +/// // ... +/// +/// Ok(()) +/// } +/// # assert_eq!(f(), Err(EINVAL)); +/// ``` +/// +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator +pub fn to_result(err: crate::ffi::c_int) -> Result { + if err < 0 { + Err(Error::from_errno(err)) + } else { + Ok(()) + } +} + +/// Transform a kernel "error pointer" to a normal pointer. +/// +/// Some kernel C API functions return an "error pointer" which optionally +/// embeds an `errno`. Callers are supposed to check the returned pointer +/// for errors. This function performs the check and converts the "error pointer" +/// to a normal pointer in an idiomatic fashion. +/// +/// Note that a `NULL` pointer is not considered an error pointer, and is returned +/// as-is, wrapped in [`Ok`]. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_err_ptr; +/// # use kernel::bindings; +/// fn devm_platform_ioremap_resource( +/// pdev: &mut PlatformDevice, +/// index: u32, +/// ) -> Result<*mut kernel::ffi::c_void> { +/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements +/// // on `index`. +/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) +/// } +/// ``` +/// +/// ``` +/// # use kernel::error::from_err_ptr; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn einval_err_ptr() -> *mut kernel::ffi::c_void { +/// # EINVAL.to_ptr() +/// # } +/// # pub(super) unsafe fn null_ptr() -> *mut kernel::ffi::c_void { +/// # core::ptr::null_mut() +/// # } +/// # pub(super) unsafe fn non_null_ptr() -> *mut kernel::ffi::c_void { +/// # 0x1234 as *mut kernel::ffi::c_void +/// # } +/// # } +/// // SAFETY: ... +/// let einval_err = from_err_ptr(unsafe { bindings::einval_err_ptr() }); +/// assert_eq!(einval_err, Err(EINVAL)); +/// +/// // SAFETY: ... +/// let null_ok = from_err_ptr(unsafe { bindings::null_ptr() }); +/// assert_eq!(null_ok, Ok(core::ptr::null_mut())); +/// +/// // SAFETY: ... +/// let non_null = from_err_ptr(unsafe { bindings::non_null_ptr() }).unwrap(); +/// assert_ne!(non_null, core::ptr::null_mut()); +/// ``` +pub fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { + // CAST: Casting a pointer to `*const crate::ffi::c_void` is always valid. + let const_ptr: *const crate::ffi::c_void = ptr.cast(); + // SAFETY: The FFI function does not deref the pointer. + if unsafe { bindings::IS_ERR(const_ptr) } { + // SAFETY: The FFI function does not deref the pointer. + let err = unsafe { bindings::PTR_ERR(const_ptr) }; + + #[allow(clippy::unnecessary_cast)] + // CAST: If `IS_ERR()` returns `true`, + // then `PTR_ERR()` is guaranteed to return a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`, + // which always fits in an `i16`, as per the invariant above. + // And an `i16` always fits in an `i32`. So casting `err` to + // an `i32` can never overflow, and is always valid. + // + // SAFETY: `IS_ERR()` ensures `err` is a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`. + return Err(unsafe { Error::from_errno_unchecked(err as crate::ffi::c_int) }); + } + Ok(ptr) +} + +/// Calls a closure returning a [`crate::error::Result`] and converts the result to +/// a C integer result. +/// +/// This is useful when calling Rust functions that return [`crate::error::Result`] +/// from inside `extern "C"` functions that need to return an integer error result. +/// +/// `T` should be convertible from an `i16` via `From`. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_result; +/// # use kernel::bindings; +/// unsafe extern "C" fn probe_callback( +/// pdev: *mut bindings::platform_device, +/// ) -> kernel::ffi::c_int { +/// from_result(|| { +/// let ptr = devm_alloc(pdev)?; +/// bindings::platform_set_drvdata(pdev, ptr); +/// Ok(0) +/// }) +/// } +/// ``` +pub fn from_result(f: F) -> T +where + T: From, + F: FnOnce() -> Result, +{ + match f() { + Ok(v) => v, + // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`, + // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above, + // therefore a negative `errno` always fits in an `i16` and will not overflow. + Err(e) => T::from(e.to_errno() as i16), + } +} + +/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait. +pub const VTABLE_DEFAULT_ERROR: &str = + "This function must not be called, see the #[vtable] documentation."; diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs new file mode 100644 index 000000000..cd4198fbb --- /dev/null +++ b/rust/kernel/faux.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the faux bus. +//! +//! This module provides bindings for working with faux devices in kernel modules. +//! +//! C header: [`include/linux/device/faux.h`](srctree/include/linux/device/faux.h) + +use crate::{ + bindings, + device, + prelude::*, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ptr::{ + null, + null_mut, + NonNull, // + }, +}; + +/// A faux device. +/// +/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a +/// real hardware device is not available or for testing. +/// +/// # Invariants +/// +/// The underlying `struct faux_device` is valid. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct faux_device`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self { + // SAFETY: `Device` is a transparent wrapper of `Opaque`. + unsafe { &*ptr.cast() } + } +} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct faux_device`. `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } +} + +// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`. +// The offset is guaranteed to point to a valid device field inside `faux::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev); +} + +/// The registration of a faux device. +/// +/// This type represents the registration of a [`struct faux_device`]. When an instance of this type +/// is dropped, its respective faux device will be unregistered from the system. +/// +/// # Invariants +/// +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. +/// +/// [`struct faux_device`]: srctree/include/linux/device/faux.h +pub struct Registration(NonNull); + +impl Registration { + /// Create and register a new faux device with the given name. + #[inline] + pub fn new(name: &CStr, parent: Option<&device::Device>) -> Result { + // SAFETY: + // - `name` is copied by this function into its own storage + // - `faux_ops` is safe to leave NULL according to the C API + // - `parent` can be either NULL or a pointer to a `struct device`, and `faux_device_create` + // will take a reference to `parent` using `device_add` - ensuring that it remains valid + // for the lifetime of the faux device. + let dev = unsafe { + bindings::faux_device_create( + name.as_char_ptr(), + parent.map_or(null_mut(), |p| p.as_raw()), + null(), + ) + }; + + // The above function will return either a valid device, or NULL on failure + // INVARIANT: The device will remain registered until faux_device_destroy() is called, which + // happens in our Drop implementation. + Ok(Self(NonNull::new(dev).ok_or(ENODEV)?)) + } + + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.as_ptr() + } +} + +impl AsRef> for Registration { + #[inline] + fn as_ref(&self) -> &Device { + // SAFETY: + // - The underlying `struct faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. + unsafe { Device::from_raw(self.as_raw()) } + } +} + +impl Drop for Registration { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.0` is a valid registered faux_device via our type invariants. + unsafe { bindings::faux_device_destroy(self.as_raw()) } + } +} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Send for Registration {} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Sync for Registration {} diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs new file mode 100644 index 000000000..a18f8b84f --- /dev/null +++ b/rust/kernel/firmware.rs @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Firmware abstraction +//! +//! C header: [`include/linux/firmware.h`](srctree/include/linux/firmware.h) + +use crate::{ + bindings, + device::Device, + error::to_result, + ffi, + prelude::*, + str::{CStr, CStrExt as _}, +}; +use core::ptr::NonNull; + +/// # Invariants +/// +/// One of the following: `bindings::request_firmware`, `bindings::firmware_request_nowarn`, +/// `bindings::firmware_request_platform`, `bindings::request_firmware_direct`. +struct FwFunc( + unsafe extern "C" fn( + *mut *const bindings::firmware, + *const ffi::c_char, + *mut bindings::device, + ) -> i32, +); + +impl FwFunc { + fn request() -> Self { + Self(bindings::request_firmware) + } + + fn request_nowarn() -> Self { + Self(bindings::firmware_request_nowarn) + } +} + +/// Abstraction around a C `struct firmware`. +/// +/// This is a simple abstraction around the C firmware API. Just like with the C API, firmware can +/// be requested. Once requested the abstraction provides direct access to the firmware buffer as +/// `&[u8]`. The firmware is released once [`Firmware`] is dropped. +/// +/// # Invariants +/// +/// The pointer is valid, and has ownership over the instance of `struct firmware`. +/// +/// The `Firmware`'s backing buffer is not modified. +/// +/// # Examples +/// +/// ```no_run +/// # use kernel::{device::Device, firmware::Firmware, sync::aref::ARef}; +/// # fn no_run(dev: ARef) -> Result<(), Error> { +/// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; +/// let blob = fw.data(); +/// +/// # Ok(()) +/// # } +/// ``` +pub struct Firmware(NonNull); + +impl Firmware { + fn request_internal(name: &CStr, dev: &Device, func: FwFunc) -> Result { + let mut fw: *mut bindings::firmware = core::ptr::null_mut(); + let pfw: *mut *mut bindings::firmware = &mut fw; + let pfw: *mut *const bindings::firmware = pfw.cast(); + + // SAFETY: `pfw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. + let ret = unsafe { func.0(pfw, name.as_char_ptr(), dev.as_raw()) }; + if ret != 0 { + return Err(Error::from_errno(ret)); + } + + // SAFETY: `func` not bailing out with a non-zero error code, guarantees that `fw` is a + // valid pointer to `bindings::firmware`. + Ok(Firmware(unsafe { NonNull::new_unchecked(fw) })) + } + + /// Send a firmware request and wait for it. See also `bindings::request_firmware`. + pub fn request(name: &CStr, dev: &Device) -> Result { + Self::request_internal(name, dev, FwFunc::request()) + } + + /// Send a request for an optional firmware module. See also + /// `bindings::firmware_request_nowarn`. + pub fn request_nowarn(name: &CStr, dev: &Device) -> Result { + Self::request_internal(name, dev, FwFunc::request_nowarn()) + } + + fn as_raw(&self) -> *mut bindings::firmware { + self.0.as_ptr() + } + + /// Returns the size of the requested firmware in bytes. + pub fn size(&self) -> usize { + // SAFETY: `self.as_raw()` is valid by the type invariant. + unsafe { (*self.as_raw()).size } + } + + /// Returns the requested firmware as `&[u8]`. + pub fn data(&self) -> &[u8] { + // SAFETY: `self.as_raw()` is valid by the type invariant. Additionally, + // `bindings::firmware` guarantees, if successfully requested, that + // `bindings::firmware::data` has a size of `bindings::firmware::size` bytes. + unsafe { core::slice::from_raw_parts((*self.as_raw()).data, self.size()) } + } +} + +impl Drop for Firmware { + fn drop(&mut self) { + // SAFETY: `self.as_raw()` is valid by the type invariant. + unsafe { bindings::release_firmware(self.as_raw()) }; + } +} + +/// Load firmware directly into the caller-provided `buf`. +/// +/// On success the firmware image has been copied into `buf`; the caller accesses the data +/// through `buf` itself. +/// +/// This is intentionally a stand-alone function rather than a `Firmware` constructor. For +/// the `into_buf` path, the firmware data lives in the caller's `buf`, not in a +/// kernel-owned buffer, so returning a `Firmware` would expose `Firmware::data()` as a +/// second handle aliasing `buf` (and `release_firmware()` does not free `buf` anyway). +pub fn request_into_buf(name: &CStr, dev: &Device, buf: &mut [u8]) -> Result { + // `as_mut_ptr()` on an empty slice returns a non-NULL pointer to + // memory which the loader does not own. Passing that pointer with `size == 0` + // makes the loader believe that it is buffer it allocated itself, so when + // `release_firmware()` is called, it will vfree the pointer and trigger a + // bug. Reject empty slices to avoid this situation. + if buf.is_empty() { + return Err(EINVAL); + } + + let mut fw: *const bindings::firmware = core::ptr::null(); + + // SAFETY: `&raw mut fw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. `buf` is a valid writable + // buffer of `buf.len()` bytes. + to_result(unsafe { + bindings::request_firmware_into_buf( + &raw mut fw, + name.as_char_ptr(), + dev.as_raw(), + buf.as_mut_ptr().cast(), + buf.len(), + ) + })?; + + // The firmware bytes are now in `buf`, which the caller owns, so we don't need + // the kernel to hang on to it any more. + // SAFETY: `fw` is a valid pointer returned by `request_firmware_into_buf`. + unsafe { bindings::release_firmware(fw) }; + + Ok(()) +} + +// SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, which is safe to be used from +// any thread. +unsafe impl Send for Firmware {} + +// SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, references to which are safe to +// be used from any thread. +unsafe impl Sync for Firmware {} + +/// Create firmware .modinfo entries. +/// +/// This macro is the counterpart of the C macro `MODULE_FIRMWARE()`, but instead of taking a +/// simple string literals, which is already covered by the `firmware` field of +/// [`crate::prelude::module!`], it allows the caller to pass a builder type, based on the +/// [`ModInfoBuilder`], which can create the firmware modinfo strings in a more flexible way. +/// +/// Drivers should extend the [`ModInfoBuilder`] with their own driver specific builder type. +/// +/// The `builder` argument must be a type which implements the following function. +/// +/// `const fn create(module_name: &'static CStr) -> ModInfoBuilder` +/// +/// `create` should pass the `module_name` to the [`ModInfoBuilder`] and, with the help of +/// it construct the corresponding firmware modinfo. +/// +/// Typically, such contracts would be enforced by a trait, however traits do not (yet) support +/// const functions. +/// +/// # Examples +/// +/// ``` +/// # mod module_firmware_test { +/// # use kernel::firmware; +/// # use kernel::prelude::*; +/// # +/// # struct MyModule; +/// # +/// # impl kernel::Module for MyModule { +/// # fn init(_module: &'static ThisModule) -> Result { +/// # Ok(Self) +/// # } +/// # } +/// # +/// # +/// struct Builder; +/// +/// impl Builder { +/// const DIR: &'static str = "vendor/chip/"; +/// const FILES: [&'static str; 3] = [ "foo", "bar", "baz" ]; +/// +/// const fn create(module_name: &'static kernel::str::CStr) -> firmware::ModInfoBuilder { +/// let mut builder = firmware::ModInfoBuilder::new(module_name); +/// +/// let mut i = 0; +/// while i < Self::FILES.len() { +/// builder = builder.new_entry() +/// .push(Self::DIR) +/// .push(Self::FILES[i]) +/// .push(".bin"); +/// +/// i += 1; +/// } +/// +/// builder +/// } +/// } +/// +/// module! { +/// type: MyModule, +/// name: "module_firmware_test", +/// authors: ["Rust for Linux"], +/// description: "module_firmware! test module", +/// license: "GPL", +/// } +/// +/// kernel::module_firmware!(Builder); +/// # } +/// ``` +#[macro_export] +macro_rules! module_firmware { + // The argument is the builder type without the const generic, since it's deferred from within + // this macro. Hence, we can neither use `expr` nor `ty`. + ($($builder:tt)*) => { + const _: () = { + const __MODULE_FIRMWARE_PREFIX: &'static $crate::str::CStr = if cfg!(MODULE) { + c"" + } else { + ::NAME + }; + + #[link_section = ".modinfo"] + #[used(compiler)] + static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX) + .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build(); + }; + }; +} + +/// Builder for firmware module info. +/// +/// [`ModInfoBuilder`] is a helper component to flexibly compose firmware paths strings for the +/// .modinfo section in const context. +/// +/// Therefore the [`ModInfoBuilder`] provides the methods [`ModInfoBuilder::new_entry`] and +/// [`ModInfoBuilder::push`], where the latter is used to push path components and the former to +/// mark the beginning of a new path string. +/// +/// [`ModInfoBuilder`] is meant to be used in combination with [`kernel::module_firmware!`]. +/// +/// The const generic `N` as well as the `module_name` parameter of [`ModInfoBuilder::new`] is an +/// internal implementation detail and supplied through the above macro. +pub struct ModInfoBuilder { + buf: [u8; N], + n: usize, + module_name: &'static CStr, +} + +impl ModInfoBuilder { + /// Create an empty builder instance. + pub const fn new(module_name: &'static CStr) -> Self { + Self { + buf: [0; N], + n: 0, + module_name, + } + } + + const fn push_internal(mut self, bytes: &[u8]) -> Self { + let mut j = 0; + + if N == 0 { + self.n += bytes.len(); + return self; + } + + while j < bytes.len() { + if self.n < N { + self.buf[self.n] = bytes[j]; + } + self.n += 1; + j += 1; + } + self + } + + /// Push an additional path component. + /// + /// Append path components to the [`ModInfoBuilder`] instance. Paths need to be separated + /// with [`ModInfoBuilder::new_entry`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::firmware::ModInfoBuilder; + /// + /// # const DIR: &str = "vendor/chip/"; + /// # const fn no_run(builder: ModInfoBuilder) { + /// let builder = builder.new_entry() + /// .push(DIR) + /// .push("foo.bin") + /// .new_entry() + /// .push(DIR) + /// .push("bar.bin"); + /// # } + /// ``` + pub const fn push(self, s: &str) -> Self { + // Check whether there has been an initial call to `next_entry()`. + if N != 0 && self.n == 0 { + crate::build_error!("Must call next_entry() before push()."); + } + + self.push_internal(s.as_bytes()) + } + + const fn push_module_name(self) -> Self { + let mut this = self; + let module_name = this.module_name; + + if !this.module_name.is_empty() { + this = this.push_internal(module_name.to_bytes_with_nul()); + + if N != 0 { + // Re-use the space taken by the NULL terminator and swap it with the '.' separator. + this.buf[this.n - 1] = b'.'; + } + } + + this + } + + /// Prepare the [`ModInfoBuilder`] for the next entry. + /// + /// This method acts as a separator between module firmware path entries. + /// + /// Must be called before constructing a new entry with subsequent calls to + /// [`ModInfoBuilder::push`]. + /// + /// See [`ModInfoBuilder::push`] for an example. + pub const fn new_entry(self) -> Self { + self.push_internal(b"\0") + .push_module_name() + .push_internal(b"firmware=") + } + + /// Build the byte array. + pub const fn build(self) -> [u8; N] { + // Add the final NULL terminator. + let this = self.push_internal(b"\0"); + + if this.n == N { + this.buf + } else { + crate::build_error!("Length mismatch."); + } + } +} + +impl ModInfoBuilder<0> { + /// Return the length of the byte array to build. + pub const fn build_length(self) -> usize { + // Compensate for the NULL terminator added by `build`. + self.n + 1 + } +} diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs new file mode 100644 index 000000000..29582b053 --- /dev/null +++ b/rust/kernel/fmt.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Formatting utilities. +//! +//! This module is intended to be used in place of `core::fmt` in kernel code. + +use kernel::prelude::*; + +pub use core::fmt::{ + Arguments, + Debug, + Error, + Formatter, + Result, + Write, // +}; + +/// Internal adapter used to route and allow implementations of formatting traits for foreign types. +/// +/// It is inserted automatically by the [`fmt!`] macro and is not meant to be used directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +#[doc(hidden)] +pub struct Adapter(pub T); + +macro_rules! impl_fmt_adapter_forward { + ($($trait:ident),* $(,)?) => { + $( + impl $trait for Adapter { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + $trait::fmt(t, f) + } + } + )* + }; +} + +use core::fmt::{ + Binary, + LowerExp, + LowerHex, + Octal, + UpperExp, + UpperHex, // +}; +use core::ptr::NonNull; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Pointer`] that allows implementing pointer formatting for foreign types. +/// +/// Together with the [`Adapter`] type and [`fmt!`] macro, it enables raw pointer formatting to be +/// intercepted and routed to [`HashedPtr`] (kernel's `%p` hashed format), preventing kernel address +/// leaks. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Pointer { + /// Same as [`core::fmt::Pointer::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +/// A wrapper for pointers that formats them using kernel's `%p` format specifier. +/// +/// By default, `%p` prints a hashed representation of the pointer address to prevent kernel address +/// leaks. When the `no_hash_pointers` kernel command-line parameter is enabled, the real address is +/// printed instead (for debugging purposes). +pub struct HashedPtr(pub *const T); + +impl Pointer for HashedPtr { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + use crate::str::CStrExt as _; + + let mut buf = [0u8; 32]; + + // Use `%#0*p` for the `0x` prefix and zero-padding; `+2` compensates for + // the prefix counting toward the field width. + let default_width = (2 * size_of::() + 2) as c_int; + let width = match (f.sign_aware_zero_pad(), f.width()) { + (true, Some(w)) if w > 0 => w.min(buf.len() - 1) as c_int, + _ => default_width, + }; + + // SAFETY: `buf` is a valid, writable 32-byte buffer, sufficient for + // all architectures (max 19 bytes for 64-bit under the default width). + // The format string is null-terminated; `width` (c_int) and pointer + // match the `%*` and `%p` specifiers. + let len = unsafe { + crate::bindings::scnprintf( + buf.as_mut_ptr().cast(), + buf.len(), + c"%#0*p".as_char_ptr(), + width, + self.0.cast::(), + ) + }; + + // SAFETY: `%#0*p` produces only ASCII, which is valid UTF-8. + let s = unsafe { core::str::from_utf8_unchecked(&buf[..len as usize]) }; + + if f.sign_aware_zero_pad() { + // `scnprintf` already applied the width and zero-padding via `%#0*p`. + f.write_str(s) + } else { + f.pad(s) + } + } +} + +// Raw pointers are formatted via `HashedPtr` (kernel `%p`: hashed by default, plain with +// `no_hash_pointers`). +impl Pointer for *const T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for *mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for &T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for &mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(core::ptr::from_ref(*self)), f) + } +} + +impl Pointer for NonNull { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(self.as_ptr()), f) + } +} + +// `Adapter<&T>` bridges our `Pointer` trait to `core::fmt::Pointer` +impl core::fmt::Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} + +/// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. +/// +/// Types should implement this trait rather than [`core::fmt::Display`]. Together with the +/// [`Adapter`] type and [`fmt!`] macro, it allows for formatting foreign types (e.g. types from +/// core) which do not implement [`core::fmt::Display`] directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Display { + /// Same as [`core::fmt::Display::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +impl Display for &T { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Display::fmt(*self, f) + } +} + +impl core::fmt::Display for Adapter<&T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + Display::fmt(t, f) + } +} + +macro_rules! impl_display_forward { + ($( + $( { $($generics:tt)* } )? $ty:ty $( { where $($where:tt)* } )? + ),* $(,)?) => { + $( + impl$($($generics)*)? Display for $ty $(where $($where)*)? { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + core::fmt::Display::fmt(self, f) + } + } + )* + }; +} + +impl_display_forward!( + bool, + char, + core::panic::PanicInfo<'_>, + Arguments<'_>, + i128, + i16, + i32, + i64, + i8, + isize, + str, + u128, + u16, + u32, + u64, + u8, + usize, + {} crate::sync::Arc {where crate::sync::Arc: core::fmt::Display}, + {} crate::sync::UniqueArc {where crate::sync::UniqueArc: core::fmt::Display}, +); + +#[macros::kunit_tests(rust_kernel_fmt)] +mod tests { + use crate::{ + bindings, + prelude::fmt, + str::CString, // + }; + + #[cfg(CONFIG_64BIT)] + mod expected { + pub(super) const PTR_VALUE: usize = 0xffffffffdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(____ptrval____)"; + pub(super) const HASHED_PREFIX: &str = "0x00000000"; + pub(super) const RAW_POINTER: &str = "0xffffffffdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xffffffffdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x000000ffffffffdeadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000ffffffffdeadbeef"; + } + + #[cfg(not(CONFIG_64BIT))] + mod expected { + pub(super) const PTR_VALUE: usize = 0xdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(ptrval)"; + pub(super) const HASHED_PREFIX: &str = "0x"; + pub(super) const RAW_POINTER: &str = "0xdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x00000000000000deadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000000000000deadbeef"; + } + + #[test] + fn test_ptr_formatting() -> core::result::Result<(), crate::error::Error> { + let ptr: *const u8 = core::ptr::without_provenance(expected::PTR_VALUE); + + // SAFETY: `no_hash_pointers` is a global variable that is never concurrently modified — + // KUnit tests may run at boot (before `mark_readonly()`) or manually afterwards (when the + // variable is read-only). Reading is always safe. + let no_hash = unsafe { bindings::no_hash_pointers }; + + if no_hash { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::PADDED_RIGHT); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::ZERO_PADDED); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::CLAMPED); + } else { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + let formatted = cstr.to_str()?; + // If the RNG is not yet ready, `%p` falls back to a placeholder. + if formatted == expected::PTR_VAL_NO_CRNG { + return Ok(()); + } + assert!(formatted.starts_with(expected::HASHED_PREFIX)); + assert_ne!(formatted, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_PADDED_RIGHT_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_ZERO_PADDED_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + let output = cstr.to_str()?; + assert!(output.starts_with("0x")); + assert!(!output[2..].chars().all(|c| c == '0')); + } + + Ok(()) + } +} diff --git a/rust/kernel/fs.rs b/rust/kernel/fs.rs new file mode 100644 index 000000000..6ba6bdf14 --- /dev/null +++ b/rust/kernel/fs.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel file systems. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) + +pub mod file; +pub use self::file::{File, LocalFile}; + +mod kiocb; +pub use self::kiocb::Kiocb; diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs new file mode 100644 index 000000000..23ee689bd --- /dev/null +++ b/rust/kernel/fs/file.rs @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Files and file descriptors. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) and +//! [`include/linux/file.h`](srctree/include/linux/file.h) + +use crate::{ + bindings, + cred::Credential, + error::{code::*, to_result, Error, Result}, + fmt, + sync::aref::{ARef, AlwaysRefCounted}, + types::{NotThreadSafe, Opaque}, +}; +use core::ptr; + +/// Primitive type representing the offset within a [`File`]. +/// +/// Type alias for `bindings::loff_t`. +pub type Offset = bindings::loff_t; + +/// Flags associated with a [`File`]. +pub mod flags { + /// File is opened in append mode. + pub const O_APPEND: u32 = bindings::O_APPEND; + + /// Signal-driven I/O is enabled. + pub const O_ASYNC: u32 = bindings::FASYNC; + + /// Close-on-exec flag is set. + pub const O_CLOEXEC: u32 = bindings::O_CLOEXEC; + + /// File was created if it didn't already exist. + pub const O_CREAT: u32 = bindings::O_CREAT; + + /// Direct I/O is enabled for this file. + pub const O_DIRECT: u32 = bindings::O_DIRECT; + + /// File must be a directory. + pub const O_DIRECTORY: u32 = bindings::O_DIRECTORY; + + /// Like [`O_SYNC`] except metadata is not synced. + pub const O_DSYNC: u32 = bindings::O_DSYNC; + + /// Ensure that this file is created with the `open(2)` call. + pub const O_EXCL: u32 = bindings::O_EXCL; + + /// Large file size enabled (`off64_t` over `off_t`). + pub const O_LARGEFILE: u32 = bindings::O_LARGEFILE; + + /// Do not update the file last access time. + pub const O_NOATIME: u32 = bindings::O_NOATIME; + + /// File should not be used as process's controlling terminal. + pub const O_NOCTTY: u32 = bindings::O_NOCTTY; + + /// If basename of path is a symbolic link, fail open. + pub const O_NOFOLLOW: u32 = bindings::O_NOFOLLOW; + + /// File is using nonblocking I/O. + pub const O_NONBLOCK: u32 = bindings::O_NONBLOCK; + + /// File is using nonblocking I/O. + /// + /// This is effectively the same flag as [`O_NONBLOCK`] on all architectures + /// except SPARC64. + pub const O_NDELAY: u32 = bindings::O_NDELAY; + + /// Used to obtain a path file descriptor. + pub const O_PATH: u32 = bindings::O_PATH; + + /// Write operations on this file will flush data and metadata. + pub const O_SYNC: u32 = bindings::O_SYNC; + + /// This file is an unnamed temporary regular file. + pub const O_TMPFILE: u32 = bindings::O_TMPFILE; + + /// File should be truncated to length 0. + pub const O_TRUNC: u32 = bindings::O_TRUNC; + + /// Bitmask for access mode flags. + /// + /// # Examples + /// + /// ``` + /// use kernel::fs::file; + /// # fn do_something() {} + /// # let flags = 0; + /// if (flags & file::flags::O_ACCMODE) == file::flags::O_RDONLY { + /// do_something(); + /// } + /// ``` + pub const O_ACCMODE: u32 = bindings::O_ACCMODE; + + /// File is read only. + pub const O_RDONLY: u32 = bindings::O_RDONLY; + + /// File is write only. + pub const O_WRONLY: u32 = bindings::O_WRONLY; + + /// File can be both read and written. + pub const O_RDWR: u32 = bindings::O_RDWR; +} + +/// Wraps the kernel's `struct file`. Thread safe. +/// +/// This represents an open file rather than a file on a filesystem. Processes generally reference +/// open files using file descriptors. However, file descriptors are not the same as files. A file +/// descriptor is just an integer that corresponds to a file, and a single file may be referenced +/// by multiple file descriptors. +/// +/// # Refcounting +/// +/// Instances of this type are reference-counted. The reference count is incremented by the +/// `fget`/`get_file` functions and decremented by `fput`. The Rust type `ARef` represents a +/// pointer that owns a reference count on the file. +/// +/// Whenever a process opens a file descriptor (fd), it stores a pointer to the file in its fd +/// table (`struct files_struct`). This pointer owns a reference count to the file, ensuring the +/// file isn't prematurely deleted while the file descriptor is open. In Rust terminology, the +/// pointers in `struct files_struct` are `ARef` pointers. +/// +/// ## Light refcounts +/// +/// Whenever a process has an fd to a file, it may use something called a "light refcount" as a +/// performance optimization. Light refcounts are acquired by calling `fdget` and released with +/// `fdput`. The idea behind light refcounts is that if the fd is not closed between the calls to +/// `fdget` and `fdput`, then the refcount cannot hit zero during that time, as the `struct +/// files_struct` holds a reference until the fd is closed. This means that it's safe to access the +/// file even if `fdget` does not increment the refcount. +/// +/// The requirement that the fd is not closed during a light refcount applies globally across all +/// threads - not just on the thread using the light refcount. For this reason, light refcounts are +/// only used when the `struct files_struct` is not shared with other threads, since this ensures +/// that other unrelated threads cannot suddenly start using the fd and close it. Therefore, +/// calling `fdget` on a shared `struct files_struct` creates a normal refcount instead of a light +/// refcount. +/// +/// Light reference counts must be released with `fdput` before the system call returns to +/// userspace. This means that if you wait until the current system call returns to userspace, then +/// all light refcounts that existed at the time have gone away. +/// +/// ### The file position +/// +/// Each `struct file` has a position integer, which is protected by the `f_pos_lock` mutex. +/// However, if the `struct file` is not shared, then the kernel may avoid taking the lock as a +/// performance optimization. +/// +/// The condition for avoiding the `f_pos_lock` mutex is different from the condition for using +/// `fdget`. With `fdget`, you may avoid incrementing the refcount as long as the current fd table +/// is not shared; it is okay if there are other fd tables that also reference the same `struct +/// file`. However, `fdget_pos` can only avoid taking the `f_pos_lock` if the entire `struct file` +/// is not shared, as different processes with an fd to the same `struct file` share the same +/// position. +/// +/// To represent files that are not thread safe due to this optimization, the [`LocalFile`] type is +/// used. +/// +/// ## Rust references +/// +/// The reference type `&File` is similar to light refcounts: +/// +/// * `&File` references don't own a reference count. They can only exist as long as the reference +/// count stays positive, and can only be created when there is some mechanism in place to ensure +/// this. +/// +/// * The Rust borrow-checker normally ensures this by enforcing that the `ARef` from which +/// a `&File` is created outlives the `&File`. +/// +/// * Using the unsafe [`File::from_raw_file`] means that it is up to the caller to ensure that the +/// `&File` only exists while the reference count is positive. +/// +/// * You can think of `fdget` as using an fd to look up an `ARef` in the `struct +/// files_struct` and create an `&File` from it. The "fd cannot be closed" rule is like the Rust +/// rule "the `ARef` must outlive the `&File`". +/// +/// # Invariants +/// +/// * All instances of this type are refcounted using the `f_count` field. +/// * There must not be any active calls to `fdget_pos` on this file that did not take the +/// `f_pos_lock` mutex. +#[repr(transparent)] +pub struct File { + inner: Opaque, +} + +// SAFETY: This file is known to not have any active `fdget_pos` calls that did not take the +// `f_pos_lock` mutex, so it is safe to transfer it between threads. +unsafe impl Send for File {} + +// SAFETY: This file is known to not have any active `fdget_pos` calls that did not take the +// `f_pos_lock` mutex, so it is safe to access its methods from several threads in parallel. +unsafe impl Sync for File {} + +// SAFETY: The type invariants guarantee that `File` is always ref-counted. This implementation +// makes `ARef` own a normal refcount. +unsafe impl AlwaysRefCounted for File { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_file(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: To call this method, the caller passes us ownership of a normal refcount, so we + // may drop it. The cast is okay since `File` has the same representation as `struct file`. + unsafe { bindings::fput(obj.cast().as_ptr()) } + } +} + +/// Wraps the kernel's `struct file`. Not thread safe. +/// +/// This type represents a file that is not known to be safe to transfer across thread boundaries. +/// To obtain a thread-safe [`File`], use the [`assume_no_fdget_pos`] conversion. +/// +/// See the documentation for [`File`] for more information. +/// +/// # Invariants +/// +/// * All instances of this type are refcounted using the `f_count` field. +/// * If there is an active call to `fdget_pos` that did not take the `f_pos_lock` mutex, then it +/// must be on the same thread as this file. +/// +/// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos +#[repr(transparent)] +pub struct LocalFile { + inner: Opaque, +} + +// SAFETY: The type invariants guarantee that `LocalFile` is always ref-counted. This implementation +// makes `ARef` own a normal refcount. +unsafe impl AlwaysRefCounted for LocalFile { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_file(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: To call this method, the caller passes us ownership of a normal refcount, so we + // may drop it. The cast is okay since `LocalFile` has the same representation as + // `struct file`. + unsafe { bindings::fput(obj.cast().as_ptr()) } + } +} + +impl LocalFile { + /// Constructs a new `struct file` wrapper from a file descriptor. + /// + /// The file descriptor belongs to the current process, and there might be active local calls + /// to `fdget_pos` on the same file. + /// + /// To obtain an `ARef`, use the [`assume_no_fdget_pos`] function to convert. + /// + /// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos + #[inline] + pub fn fget(fd: u32) -> Result, BadFdError> { + // SAFETY: FFI call, there are no requirements on `fd`. + let ptr = ptr::NonNull::new(unsafe { bindings::fget(fd) }).ok_or(BadFdError)?; + + // SAFETY: `bindings::fget` created a refcount, and we pass ownership of it to the `ARef`. + // + // INVARIANT: This file is in the fd table on this thread, so either all `fdget_pos` calls + // are on this thread, or the file is shared, in which case `fdget_pos` calls took the + // `f_pos_lock` mutex. + Ok(unsafe { ARef::from_raw(ptr.cast()) }) + } + + /// Creates a reference to a [`LocalFile`] from a valid pointer. + /// + /// # Safety + /// + /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is + /// positive for the duration of `'a`. + /// * The caller must ensure that if there is an active call to `fdget_pos` that did not take + /// the `f_pos_lock` mutex, then that call is on the current thread. + #[inline] + pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a LocalFile { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of `'a`. The cast is okay because `LocalFile` is `repr(transparent)`. + // + // INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls. + unsafe { &*ptr.cast() } + } + + /// Assume that there are no active `fdget_pos` calls that prevent us from sharing this file. + /// + /// This makes it safe to transfer this file to other threads. No checks are performed, and + /// using it incorrectly may lead to a data race on the file position if the file is shared + /// with another thread. + /// + /// This method is intended to be used together with [`LocalFile::fget`] when the caller knows + /// statically that there are no `fdget_pos` calls on the current thread. For example, you + /// might use it when calling `fget` from an ioctl, since ioctls usually do not touch the file + /// position. + /// + /// # Safety + /// + /// There must not be any active `fdget_pos` calls on the current thread. + #[inline] + pub unsafe fn assume_no_fdget_pos(me: ARef) -> ARef { + // INVARIANT: There are no `fdget_pos` calls on the current thread, and by the type + // invariants, if there is a `fdget_pos` call on another thread, then it took the + // `f_pos_lock` mutex. + // + // SAFETY: `LocalFile` and `File` have the same layout. + unsafe { ARef::from_raw(ARef::into_raw(me).cast()) } + } + + /// Returns a raw pointer to the inner C struct. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::file { + self.inner.get() + } + + /// Returns the credentials of the task that originally opened the file. + pub fn cred(&self) -> &Credential { + // SAFETY: It's okay to read the `f_cred` field without synchronization because `f_cred` is + // never changed after initialization of the file. + let ptr = unsafe { (*self.as_ptr()).f_cred }; + + // SAFETY: The signature of this function ensures that the caller will only access the + // returned credential while the file is still valid, and the C side ensures that the + // credential stays valid at least as long as the file. + unsafe { Credential::from_ptr(ptr) } + } + + /// Returns the flags associated with the file. + /// + /// The flags are a combination of the constants in [`flags`]. + #[inline] + pub fn flags(&self) -> u32 { + // This `read_volatile` is intended to correspond to a READ_ONCE call. + // + // SAFETY: The file is valid because the shared reference guarantees a nonzero refcount. + // + // FIXME(read_once): Replace with `read_once` when available on the Rust side. + unsafe { core::ptr::addr_of!((*self.as_ptr()).f_flags).read_volatile() } + } +} + +impl File { + /// Creates a reference to a [`File`] from a valid pointer. + /// + /// # Safety + /// + /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is + /// positive for the duration of `'a`. + /// * The caller must ensure that if there are active `fdget_pos` calls on this file, then they + /// took the `f_pos_lock` mutex. + #[inline] + pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a File { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of `'a`. The cast is okay because `File` is `repr(transparent)`. + // + // INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls. + unsafe { &*ptr.cast() } + } +} + +// Make LocalFile methods available on File. +impl core::ops::Deref for File { + type Target = LocalFile; + #[inline] + fn deref(&self) -> &LocalFile { + // SAFETY: The caller provides a `&File`, and since it is a reference, it must point at a + // valid file for the desired duration. + // + // By the type invariants, there are no `fdget_pos` calls that did not take the + // `f_pos_lock` mutex. + unsafe { LocalFile::from_raw_file(core::ptr::from_ref(self).cast()) } + } +} + +/// A file descriptor reservation. +/// +/// This allows the creation of a file descriptor in two steps: first, we reserve a slot for it, +/// then we commit or drop the reservation. The first step may fail (e.g., the current process ran +/// out of available slots), but commit and drop never fail (and are mutually exclusive). +/// +/// Dropping the reservation happens in the destructor of this type. +/// +/// # Invariants +/// +/// The fd stored in this struct must correspond to a reserved file descriptor of the current task. +pub struct FileDescriptorReservation { + fd: u32, + /// Prevent values of this type from being moved to a different task. + /// + /// The `fd_install` and `put_unused_fd` functions assume that the value of `current` is + /// unchanged since the call to `get_unused_fd_flags`. By adding this marker to this type, we + /// prevent it from being moved across task boundaries, which ensures that `current` does not + /// change while this value exists. + _not_send: NotThreadSafe, +} + +impl FileDescriptorReservation { + /// Creates a new file descriptor reservation. + #[inline] + pub fn get_unused_fd_flags(flags: u32) -> Result { + // SAFETY: FFI call, there are no safety requirements on `flags`. + let fd: i32 = unsafe { bindings::get_unused_fd_flags(flags) }; + to_result(fd)?; + + Ok(Self { + fd: fd as u32, + _not_send: NotThreadSafe, + }) + } + + /// Returns the file descriptor number that was reserved. + #[inline] + pub fn reserved_fd(&self) -> u32 { + self.fd + } + + /// Commits the reservation. + /// + /// The previously reserved file descriptor is bound to `file`. This method consumes the + /// [`FileDescriptorReservation`], so it will not be usable after this call. + #[inline] + pub fn fd_install(self, file: ARef) { + // SAFETY: `self.fd` was previously returned by `get_unused_fd_flags`. We have not yet used + // the fd, so it is still valid, and `current` still refers to the same task, as this type + // cannot be moved across task boundaries. + // + // Furthermore, the file pointer is guaranteed to own a refcount by its type invariants, + // and we take ownership of that refcount by not running the destructor below. + // Additionally, the file is known to not have any non-shared `fdget_pos` calls, so even if + // this process starts using the file position, this will not result in a data race on the + // file position. + unsafe { bindings::fd_install(self.fd, file.as_ptr()) }; + + // `fd_install` consumes both the file descriptor and the file reference, so we cannot run + // the destructors. + core::mem::forget(self); + core::mem::forget(file); + } +} + +impl Drop for FileDescriptorReservation { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants of this type, `self.fd` was previously returned by + // `get_unused_fd_flags`. We have not yet used the fd, so it is still valid, and `current` + // still refers to the same task, as this type cannot be moved across task boundaries. + unsafe { bindings::put_unused_fd(self.fd) }; + } +} + +/// Represents the [`EBADF`] error code. +/// +/// Used for methods that can only fail with [`EBADF`]. +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct BadFdError; + +impl From for Error { + #[inline] + fn from(_: BadFdError) -> Error { + EBADF + } +} + +impl fmt::Debug for BadFdError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("EBADF") + } +} diff --git a/rust/kernel/fs/kiocb.rs b/rust/kernel/fs/kiocb.rs new file mode 100644 index 000000000..84c936cd6 --- /dev/null +++ b/rust/kernel/fs/kiocb.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Kernel IO callbacks. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) + +use core::marker::PhantomData; +use core::ptr::NonNull; +use kernel::types::ForeignOwnable; + +/// Wrapper for the kernel's `struct kiocb`. +/// +/// Currently this abstractions is incomplete and is essentially just a tuple containing a +/// reference to a file and a file position. +/// +/// The type `T` represents the filesystem or driver specific data associated with the file. +/// +/// # Invariants +/// +/// `inner` points at a valid `struct kiocb` whose file has the type `T` as its private data. +pub struct Kiocb<'a, T> { + inner: NonNull, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T: ForeignOwnable> Kiocb<'a, T> { + /// Create a `Kiocb` from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference a valid `struct kiocb` for the duration of `'a`. The private + /// data of the file must be `T`. + pub unsafe fn from_raw(kiocb: *mut bindings::kiocb) -> Self { + Self { + // SAFETY: If a pointer is valid it is not null. + inner: unsafe { NonNull::new_unchecked(kiocb) }, + _phantom: PhantomData, + } + } + + /// Access the underlying `struct kiocb` directly. + pub fn as_raw(&self) -> *mut bindings::kiocb { + self.inner.as_ptr() + } + + /// Get the filesystem or driver specific data associated with the file. + pub fn file(&self) -> ::Borrowed<'a> { + // SAFETY: We have shared access to this kiocb and hence the underlying file, so we can + // read the file's private data. + let private = unsafe { (*(*self.as_raw()).ki_filp).private_data }; + // SAFETY: The kiocb has shared access to the private data. + unsafe { ::borrow(private) } + } + + /// Gets the current value of `ki_pos`. + pub fn ki_pos(&self) -> i64 { + // SAFETY: We have shared access to the kiocb, so we can read its `ki_pos` field. + unsafe { (*self.as_raw()).ki_pos } + } + + /// Gets a mutable reference to the `ki_pos` field. + pub fn ki_pos_mut(&mut self) -> &mut i64 { + // SAFETY: We have exclusive access to the kiocb, so we can write to `ki_pos`. + unsafe { &mut (*self.as_raw()).ki_pos } + } +} diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs new file mode 100644 index 000000000..f29244fb0 --- /dev/null +++ b/rust/kernel/fwctl.rs @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the fwctl subsystem. +//! +//! C header: `include/linux/fwctl.h` + +use crate::{ + bindings, + container_of, + device, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + ptr::NonNull, + slice, // +}; + +/// Returns a kmalloc-compatible allocation size for `T`. +const fn kmalloc_aligned_size() -> usize { + Layout::new::().pad_to_align().size() +} + +/// Represents a fwctl device type. +/// +/// Corresponds to the C `enum fwctl_device_type`. All non-error UAPI values are represented so +/// Rust drivers can select a device type without passing an untyped integer, while +/// `FWCTL_DEVICE_TYPE_ERROR` remains unrepresentable. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum DeviceType { + /// Mellanox ConnectX (mlx5) device. + Mlx5 = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_MLX5, + /// CXL (Compute Express Link) device. + Cxl = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_CXL, + /// AMD/Pensando PDS device. + Pds = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_PDS, + /// Broadcom NetXtreme (bnxt) device. + Bnxt = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_BNXT, +} + +/// Scope of access for an RPC request. +/// +/// Corresponds to the C `enum fwctl_rpc_scope`. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RpcScope { + /// Read/write access to device configuration. + Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION, + /// Read-only access to debug information. + DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY, + /// Write access to lockdown-compatible debug information. + DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE, + /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`). + DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL, +} + +impl TryFrom for RpcScope { + type Error = Error; + + #[inline] + fn try_from(value: u32) -> Result { + match value { + v if v == Self::Configuration as u32 => Ok(Self::Configuration), + v if v == Self::DebugReadOnly as u32 => Ok(Self::DebugReadOnly), + v if v == Self::DebugWrite as u32 => Ok(Self::DebugWrite), + v if v == Self::DebugWriteFull as u32 => Ok(Self::DebugWriteFull), + _ => Err(EINVAL), + } + } +} + +/// Response from a [`Operations::fw_rpc`] call. +pub enum FwRpcResponse { + /// Reuse the input buffer as the output, with the given output length. + /// + /// The callback returns `EINVAL` if the output length exceeds the input buffer length. + InPlace(usize), + /// Return a newly allocated buffer as the output. + NewBuffer(KVVec), +} + +/// Trait implemented by each Rust driver that integrates with the fwctl subsystem. +/// +/// The implementing type **is** the per-FD user context: one instance is +/// created for each `open()` call and dropped when the FD is closed. +/// +/// Each implementation corresponds to a specific device type and provides the +/// vtable used by the core `fwctl` layer to manage per-FD user contexts and +/// handle RPC requests. +pub trait Operations: Sized + Send + Sync + 'static { + /// Data owned by the [`Registration`] and accessible during callbacks. + /// + /// The lifetime `'a` is tied to the [`Registration`] scope (which lives within the parent bus + /// device binding scope). Drivers use it to store references to resources bound to this scope, + /// such as PCI BARs or typed bus device references. + type RegistrationData<'a>: Send + Sync + 'a + where + Self: 'a; + + /// fwctl device type identifier. + const DEVICE_TYPE: DeviceType; + + /// Called when a new user context is opened. + /// + /// Returns a [`PinInit`] initializer for `Self`. The instance is dropped + /// automatically when the FD is closed (after [`close`](Self::close)). + fn open<'a>( + device: &Device, + reg_data: &Self::RegistrationData<'a>, + ) -> impl PinInit; + + /// Called when the user context is closed. + /// + /// The driver may perform additional cleanup here that requires access + /// to the owning [`Device`]. `Self` is dropped automatically after this + /// returns. + fn close<'a>( + _this: Pin<&mut Self>, + _device: &Device, + _reg_data: &Self::RegistrationData<'a>, + ) { + } + + /// Return device information to userspace. + /// + /// The default implementation returns no device-specific data. + fn info<'a>( + _this: Pin<&Self>, + _device: &Device, + _reg_data: &Self::RegistrationData<'a>, + ) -> Result, Error> { + Ok(KVec::new()) + } + + /// Handle a userspace RPC request. + /// + /// `max_output_len` is the size of the userspace output buffer. A driver may return a larger + /// response to report the required size; the fwctl core copies only the bytes that fit and + /// reports the full response length to userspace. + fn fw_rpc<'a>( + this: Pin<&Self>, + device: &Device, + reg_data: &Self::RegistrationData<'a>, + scope: RpcScope, + rpc_buf: &mut [u8], + max_output_len: usize, + ) -> Result; +} + +/// A fwctl device. +/// +/// `#[repr(C)]` with the `fwctl_device` at offset 0, matching the C `fwctl_alloc_device()` layout +/// convention. Contains a pointer to the [`Registration`]'s data, set at registration time and +/// cleared on unregistration. +/// +/// # Invariants +/// +/// - `dev` is embedded at offset 0 and is initialised by fwctl. +/// - The fwctl refcount owns the allocation lifetime. +/// - `registration_data` is either [`NonNull::dangling()`] (before registration / after +/// unregistration) or points to valid data owned by the [`Registration`]. +#[repr(C)] +pub struct Device { + dev: Opaque, + registration_data: UnsafeCell>>, +} + +impl Device { + /// Allocate a new fwctl device. + /// + /// Returns an [`ARef`] that can be passed to [`Registration::new()`] + /// to make the device visible to userspace. + pub fn new(parent: &device::Device) -> Result> { + const_assert!( + core::mem::offset_of!(Self, dev) == 0, + "struct fwctl_device must be at offset 0" + ); + + let size = kmalloc_aligned_size::(); + let ops = core::ptr::from_ref::(&VTable::::VTABLE).cast_mut(); + + // SAFETY: `ops` is static, `parent` is bound, and `size` is padded so the allocation made + // by `_fwctl_alloc_device` satisfies the size and alignment required by `Device`. + let raw = unsafe { bindings::_fwctl_alloc_device(parent.as_raw(), ops, size) }; + let this = NonNull::new(raw.cast::()).ok_or(ENOMEM)?; + + // INVARIANT: Set `registration_data` to dangling (no registration yet). + // SAFETY: `this` points to the allocation just returned by fwctl. + unsafe { + (&raw mut (*this.as_ptr()).registration_data) + .write(UnsafeCell::new(NonNull::dangling())); + }; + + // SAFETY: `this` owns the initial reference. + Ok(unsafe { ARef::from_raw(this) }) + } + + /// Returns the underlying `fwctl_device` pointer. + #[inline] + fn as_raw(&self) -> *mut bindings::fwctl_device { + self.dev.get() + } + + /// Borrows a Rust fwctl device from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a valid `fwctl_device` embedded in a [`Device`]. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_device) -> &'a Self { + // SAFETY: The caller upholds the offset-0 `Device` invariant. + unsafe { &*ptr.cast() } + } + + /// Invokes `f` with the registration data. + /// + /// The higher-ranked callback prevents the erased registration lifetime from escaping and + /// permits registration data that is invariant over its lifetime parameter. + /// + /// # Safety + /// + /// The caller must ensure that the device is registered and that this is called from a fwctl + /// callback protected by `registration_lock`. + #[inline] + unsafe fn with_registration_data( + &self, + f: impl for<'a> FnOnce(&Device, &'a T::RegistrationData<'a>) -> R, + ) -> R { + // SAFETY: Caller guarantees the device is registered, so the pointer is valid. + // Lifetimes do not affect layout. The higher-ranked callback prevents the shortened + // lifetime from escaping or being selected by the caller. + let reg_data = unsafe { + (*self.registration_data.get()) + .cast::>() + .as_ref() + }; + + f(self, reg_data) + } +} + +impl AsRef for Device { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: `self` contains a live fwctl_device. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + // SAFETY: The embedded device is initialised by fwctl. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: `fwctl_get` increments the refcount of a valid fwctl_device. +// `fwctl_put` decrements it and frees the device when it reaches zero. +unsafe impl AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self` holds a live reference. + unsafe { bindings::fwctl_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller owns a live reference. + unsafe { bindings::fwctl_put(obj.cast().as_ptr()) }; + } +} + +// SAFETY: `Device` is refcounted by the fwctl core and may be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: Shared access to the embedded `fwctl_device` is protected by the fwctl core. The +// `registration_data` field is only mutated before registration and after unregistration (both +// single-threaded with respect to callbacks). +unsafe impl Sync for Device {} + +/// A registered fwctl device. +/// +/// Owns the [`RegistrationData`](Operations::RegistrationData) made available to driver callbacks. +/// The parent device lifetime ensures that [`fwctl_unregister`] runs before the parent driver +/// unbinds. +/// +/// On drop the device is unregistered (all user contexts are closed and `ops` is set to `NULL`) +/// and the registration data is dropped. +/// +/// [`fwctl_unregister`]: srctree/drivers/fwctl/main.c +pub struct Registration<'a, T: Operations> { + dev: ARef>, + _reg_data: Pin>>, +} + +impl<'a, T: Operations> Registration<'a, T> { + /// Register a previously allocated fwctl device with the given registration data. + /// + /// The `reg_data` is owned by the registration and accessible during callbacks. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since `fwctl_unregister` must be called before the + /// parent device is unbound. + /// + /// `dev` must be an unregistered [`Device`] that is not associated with any live + /// [`Registration`], and no other thread may attempt to register the same device concurrently. + pub unsafe fn new( + parent: &'a device::Device, + dev: &Device, + reg_data: impl PinInit, Error>, + ) -> Result { + let actual_parent = dev.as_ref().parent().ok_or(EINVAL)?; + let parent_device: &device::Device = parent; + if !core::ptr::eq(actual_parent, parent_device) { + return Err(EINVAL); + } + + let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once callbacks can be invoked. The `'static` type is only an erased storage + // handle; callbacks access the pointer through a higher-ranked closure. + let ptr: NonNull> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *dev.registration_data.get() = ptr }; + + // SAFETY: `dev` is a valid fwctl_device backed by an ARef. + let ret = unsafe { bindings::fwctl_register(dev.as_raw()) }; + if ret != 0 { + // SAFETY: No concurrent readers; registration failed. + unsafe { *dev.registration_data.get() = NonNull::dangling() }; + return Err(Error::from_errno(ret)); + } + + Ok(Self { + dev: dev.into(), + _reg_data: reg_data, + }) + } +} + +impl Drop for Registration<'_, T> { + fn drop(&mut self) { + // SAFETY: The Registration lifetime guarantees that the parent device is still bound. + // `fwctl_unregister` takes the write lock, closes all user contexts, and sets ops=NULL. + // After it returns, no callbacks can be running or will run. + unsafe { bindings::fwctl_unregister(self.dev.as_raw()) }; + + // SAFETY: `fwctl_unregister` guarantees no concurrent readers. + unsafe { *self.dev.registration_data.get() = NonNull::dangling() }; + + // `self._reg_data` is dropped here, after callbacks have stopped. + } +} + +/// Internal per-FD user context wrapping `struct fwctl_uctx` and `T`. +/// +/// Not exposed to drivers; they work with `&T` / `Pin<&mut T>` directly. +#[repr(C)] +#[pin_data] +struct UserCtx { + #[pin] + fwctl_uctx: Opaque, + #[pin] + uctx: T, +} + +impl UserCtx { + /// Borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx` that remains + /// valid and does not move for the duration of `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a Self> { + // SAFETY: The caller upholds the `UserCtx` embedding, lifetime, and pinning invariants. + unsafe { Pin::new_unchecked(&*container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx)) } + } + + /// Mutably borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// - `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx` that remains + /// valid and does not move for the duration of `'a`. + /// - The caller must ensure exclusive access to the `UserCtx` for the duration of `'a`. + #[inline] + unsafe fn from_raw_mut<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a mut Self> { + // SAFETY: The caller upholds the embedding, lifetime, pinning, and exclusivity invariants. + unsafe { + Pin::new_unchecked( + &mut *container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx).cast_mut(), + ) + } + } + + /// Returns a reference to the fwctl [`Device`] that owns this context. + #[inline] + fn device(self: Pin<&Self>) -> &Device { + // SAFETY: fwctl initialises this pointer before any driver callback. + let raw_fwctl = unsafe { (*self.fwctl_uctx.get()).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device` layout. + unsafe { Device::from_raw(raw_fwctl) } + } + + /// Returns a pinned reference to the driver context. + #[inline] + fn uctx(self: Pin<&Self>) -> Pin<&T> { + ::pin_init::assert_pinned!(UserCtx, uctx, T, inline); + + // SAFETY: `uctx` is structurally pinned. + unsafe { self.map_unchecked(|ctx| &ctx.uctx) } + } +} + +/// Static vtable mapping Rust trait methods to C callbacks. +struct VTable(PhantomData); + +impl VTable { + /// The fwctl operations vtable for this driver type. + const VTABLE: bindings::fwctl_ops = bindings::fwctl_ops { + // CAST: `DeviceType` has the same `u32` representation as the C enum field. + device_type: T::DEVICE_TYPE as u32, + uctx_size: kmalloc_aligned_size::>(), + open_uctx: Some(Self::open_uctx_callback), + close_uctx: Some(Self::close_uctx_callback), + info: Some(Self::info_callback), + fw_rpc: Some(Self::fw_rpc_callback), + }; + + /// Initialises a newly opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must be a valid `fwctl_uctx` embedded in a `UserCtx` with + /// sufficient allocated space for the uctx field. + unsafe extern "C" fn open_uctx_callback(uctx: *mut bindings::fwctl_uctx) -> ffi::c_int { + const_assert!( + core::mem::offset_of!(UserCtx, fwctl_uctx) == 0, + "struct fwctl_uctx must be at offset 0" + ); + + // SAFETY: fwctl sets this pointer before calling `open_uctx`. + let raw_fwctl = unsafe { (*uctx).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device` layout. + let device = unsafe { Device::::from_raw(raw_fwctl) }; + + let uctx_offset = core::mem::offset_of!(UserCtx, uctx); + // SAFETY: `uctx_size` reserves space for the full `UserCtx`. + let uctx_ptr: *mut T = unsafe { uctx.byte_add(uctx_offset).cast() }; + + // SAFETY: `open_uctx` is called under `registration_lock` read, so the device is + // registered. `uctx_ptr` addresses the uninitialised pinned context reserved by + // `uctx_size`. + unsafe { + device.with_registration_data(|device, reg_data| { + match pin_init::raw_try_init(uctx_ptr, T::open(device, reg_data)) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + }) + } + } + + /// Closes and drops an opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must point to a fully initialised `UserCtx`. + unsafe extern "C" fn close_uctx_callback(uctx: *mut bindings::fwctl_uctx) { + // SAFETY: fwctl keeps the owning device live for this callback. + let device = unsafe { Device::::from_raw((*uctx).fwctl) }; + + // SAFETY: close is called for an opened Rust user context. + let mut ctx = unsafe { UserCtx::::from_raw_mut(uctx) }; + + // SAFETY: `close_uctx` is called under `registration_lock` write (from + // `fwctl_unregister`) or read (from `fwctl_fops_release`), so the device is registered. + unsafe { + device.with_registration_data(|device, reg_data| { + T::close(ctx.as_mut().project().uctx, device, reg_data); + }); + } + + // SAFETY: close is the last callback before fwctl frees the allocation. + unsafe { core::ptr::drop_in_place(ctx.project().uctx.get_unchecked_mut()) }; + } + + /// Returns device-specific information for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx`. + /// - `length` must be a valid pointer. + unsafe extern "C" fn info_callback( + uctx: *mut bindings::fwctl_uctx, + length: *mut usize, + ) -> *mut ffi::c_void { + // SAFETY: info is called for an opened Rust user context. + let ctx = unsafe { UserCtx::::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: `info` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| T::info(ctx.uctx(), device, reg_data)) + }; + + match result { + Ok(kvec) if kvec.is_empty() => { + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = 0 }; + // Return NULL for empty data; kfree(NULL) is safe. + core::ptr::null_mut() + } + Ok(kvec) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = len }; + ptr.cast::() + } + Err(e) => Error::to_ptr(e), + } + } + + /// Dispatches a firmware RPC for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx`. + /// - `rpc_in` must be valid, initialised, and exclusively accessible for `in_len` bytes. + /// - `out_len` must be valid for reading and writing an initialised `usize`. + unsafe extern "C" fn fw_rpc_callback( + uctx: *mut bindings::fwctl_uctx, + scope: u32, + rpc_in: *mut ffi::c_void, + in_len: usize, + out_len: *mut usize, + ) -> *mut ffi::c_void { + let scope = match RpcScope::try_from(scope) { + Ok(s) => s, + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` points to an initialised `usize` supplied by fwctl. + let max_output_len = unsafe { *out_len }; + + // SAFETY: RPC is called for an opened Rust user context. + let ctx = unsafe { UserCtx::::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: fwctl passes an exclusively owned buffer that is valid and initialised for + // `in_len` bytes. It remains live for the duration of this callback. + let rpc_buf = unsafe { slice::from_raw_parts_mut(rpc_in.cast::(), in_len) }; + + // SAFETY: `fw_rpc` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| { + T::fw_rpc(ctx.uctx(), device, reg_data, scope, rpc_buf, max_output_len) + }) + }; + + let (response, response_len) = match result { + Ok(FwRpcResponse::InPlace(len)) => { + if len > in_len { + return Error::to_ptr(EINVAL); + } + + (rpc_in, len) + } + Ok(FwRpcResponse::NewBuffer(kvec)) if kvec.is_empty() => { + // Return NULL for empty data; kvfree(NULL) is safe. + (core::ptr::null_mut(), 0) + } + Ok(FwRpcResponse::NewBuffer(kvec)) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + (ptr.cast::(), len) + } + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` is a valid out-parameter. + unsafe { *out_len = response_len }; + response + } +} diff --git a/rust/kernel/generated_arch_reachable_asm.rs.S b/rust/kernel/generated_arch_reachable_asm.rs.S new file mode 100644 index 000000000..3886a9ad3 --- /dev/null +++ b/rust/kernel/generated_arch_reachable_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_REACHABLE) diff --git a/rust/kernel/generated_arch_static_branch_asm.rs.S b/rust/kernel/generated_arch_static_branch_asm.rs.S new file mode 100644 index 000000000..2afb63870 --- /dev/null +++ b/rust/kernel/generated_arch_static_branch_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_STATIC_BRANCH_ASM("{symb} + {off} + {branch}", "{l_yes}")) diff --git a/rust/kernel/generated_arch_warn_asm.rs.S b/rust/kernel/generated_arch_warn_asm.rs.S new file mode 100644 index 000000000..409eb4c2d --- /dev/null +++ b/rust/kernel/generated_arch_warn_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_ASM("{file}", "{line}", "{flags}", "{size}")) diff --git a/rust/kernel/gpu.rs b/rust/kernel/gpu.rs new file mode 100644 index 000000000..1dc5d0c8c --- /dev/null +++ b/rust/kernel/gpu.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU subsystem abstractions. + +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod buddy; diff --git a/rust/kernel/gpu/buddy.rs b/rust/kernel/gpu/buddy.rs new file mode 100644 index 000000000..d502ada6e --- /dev/null +++ b/rust/kernel/gpu/buddy.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU buddy allocator bindings. +//! +//! C header: [`include/linux/gpu_buddy.h`](srctree/include/linux/gpu_buddy.h) +//! +//! This module provides Rust abstractions over the Linux kernel's GPU buddy +//! allocator, which implements a binary buddy memory allocator. +//! +//! The buddy allocator manages a contiguous address space and allocates blocks +//! in power-of-two sizes, useful for GPU physical memory management. +//! +//! # Examples +//! +//! Create a buddy allocator and perform a basic range allocation: +//! +//! ``` +//! use kernel::{ +//! gpu::buddy::{ +//! GpuBuddy, +//! GpuBuddyAllocFlags, +//! GpuBuddyAllocMode, +//! GpuBuddyParams, // +//! }, +//! prelude::*, +//! ptr::Alignment, +//! sizes::*, // +//! }; +//! +//! // Create a 1GB buddy allocator with 4KB minimum chunk size. +//! let buddy = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_1G as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! assert_eq!(buddy.size(), SZ_1G as u64); +//! assert_eq!(buddy.chunk_size(), Alignment::new::()); +//! let initial_free = buddy.avail(); +//! +//! // Allocate 16MB. Block lands at the top of the address range. +//! let allocated = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = allocated.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); // 2^12 pages = 16MB +//! assert_eq!(block.size(), SZ_16M as u64); +//! assert_eq!(allocated.iter().count(), 1); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(allocated); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Top-down allocation allocates from the highest addresses: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{GpuBuddy, GpuBuddyAllocMode, GpuBuddyAllocFlags, GpuBuddyParams}, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! let topdown = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::TopDown, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = topdown.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); +//! assert_eq!(block.size(), SZ_16M as u64); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(topdown); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Non-contiguous allocation can fill fragmented memory by returning multiple +//! blocks: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! // Create fragmentation by allocating 4MB blocks at [0,4M) and [8M,12M). +//! let frag1 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_4M as u64); +//! +//! let frag2 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_8M as u64); +//! +//! // Allocate 8MB, this returns 2 blocks from the holes. +//! let fragmented = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_16M as u64), +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let (mut count, mut total) = (0u32, 0u64); +//! for block in fragmented.iter() { +//! assert_eq!(block.size(), SZ_4M as u64); +//! total += block.size(); +//! count += 1; +//! } +//! assert_eq!(total, SZ_8M as u64); +//! assert_eq!(count, 2); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Contiguous allocation fails when only fragmented space is available: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlag, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! // Create a small 16MB buddy allocator with fragmented memory. +//! let small = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_16M as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! let _hole1 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! let _hole2 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! // 8MB contiguous should fail, only two non-contiguous 4MB holes exist. +//! let result = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlag::Contiguous, +//! ), +//! GFP_KERNEL, +//! ); +//! assert!(result.is_err()); +//! # Ok::<(), Error>(()) +//! ``` + +use core::ops::Range; + +use crate::{ + bindings, + clist_create, + error::to_result, + interop::list::CListHead, + new_mutex, + prelude::*, + ptr::Alignment, + sync::{ + lock::mutex::MutexGuard, + Arc, + Mutex, // + }, + types::Opaque, // +}; + +/// Allocation mode for the GPU buddy allocator. +/// +/// The mode determines the primary allocation strategy. Modes are mutually +/// exclusive: an allocation is either simple, range-constrained, or top-down. +/// +/// Orthogonal modifier flags (e.g., contiguous, clear) are specified separately +/// via [`GpuBuddyAllocFlags`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GpuBuddyAllocMode { + /// Simple allocation without constraints. + Simple, + /// Range-based allocation within the given address range. + Range(Range), + /// Allocate from top of address space downward. + TopDown, +} + +impl GpuBuddyAllocMode { + /// Returns the C flags corresponding to the allocation mode. + fn as_flags(&self) -> usize { + match self { + Self::Simple => 0, + Self::Range(_) => bindings::GPU_BUDDY_RANGE_ALLOCATION, + Self::TopDown => bindings::GPU_BUDDY_TOPDOWN_ALLOCATION, + } + } + + /// Extracts the range start/end, defaulting to `(0, 0)` for non-range modes. + fn range(&self) -> (u64, u64) { + match self { + Self::Range(range) => (range.start, range.end), + _ => (0, 0), + } + } +} + +crate::impl_flags!( + /// Modifier flags for GPU buddy allocation. + /// + /// These flags can be combined with any [`GpuBuddyAllocMode`] to control + /// additional allocation behavior. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + pub struct GpuBuddyAllocFlags(usize); + + /// Individual modifier flag for GPU buddy allocation. + #[derive(Clone, Copy, PartialEq, Eq)] + pub enum GpuBuddyAllocFlag { + /// Allocate physically contiguous blocks. + Contiguous = bindings::GPU_BUDDY_CONTIGUOUS_ALLOCATION, + + /// Request allocation from cleared (zeroed) memory. + Clear = bindings::GPU_BUDDY_CLEAR_ALLOCATION, + + /// Disable trimming of partially used blocks. + TrimDisable = bindings::GPU_BUDDY_TRIM_DISABLE, + } +); + +/// Parameters for creating a GPU buddy allocator. +pub struct GpuBuddyParams { + /// Base offset (in bytes) where the managed memory region starts. + /// Allocations will be offset by this value. + pub base_offset: u64, + /// Total size (in bytes) of the address space managed by the allocator. + pub size: u64, + /// Minimum allocation unit / chunk size; must be >= 4KB. + pub chunk_size: Alignment, +} + +/// Inner structure holding the actual buddy allocator. +/// +/// # Synchronization +/// +/// The C `gpu_buddy` API requires synchronization (see `include/linux/gpu_buddy.h`). +/// Internal locking ensures all allocator and free operations are properly +/// synchronized, preventing races between concurrent allocations and the +/// freeing that occurs when [`AllocatedBlocks`] is dropped. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains an initialized buddy allocator. +#[pin_data(PinnedDrop)] +struct GpuBuddyInner { + #[pin] + inner: Opaque, + + // TODO: Replace `Mutex<()>` with `Mutex>` once `Mutex::new()` + // accepts `impl PinInit`. + #[pin] + lock: Mutex<()>, + /// Cached creation parameters (do not change after init). + params: GpuBuddyParams, +} + +impl GpuBuddyInner { + /// Create a pin-initializer for the buddy allocator. + fn new(params: GpuBuddyParams) -> impl PinInit { + let size = params.size; + let chunk_size = params.chunk_size; + + // INVARIANT: `gpu_buddy_init` returns 0 on success, at which point the + // `gpu_buddy` structure is initialized and ready for use with all + // `gpu_buddy_*` APIs. `try_pin_init!` only completes if all fields succeed, + // so the invariant holds when construction finishes. + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `ptr` points to valid uninitialized memory from the pin-init + // infrastructure. `gpu_buddy_init` will initialize the structure. + to_result(unsafe { + bindings::gpu_buddy_init(ptr, size, chunk_size.as_usize() as u64) + }) + }), + lock <- new_mutex!(()), + params, + }) + } + + /// Lock the mutex and return a guard for accessing the allocator. + fn lock(&self) -> GpuBuddyGuard<'_> { + GpuBuddyGuard { + inner: self, + _guard: self.lock.lock(), + } + } +} + +#[pinned_drop] +impl PinnedDrop for GpuBuddyInner { + fn drop(self: Pin<&mut Self>) { + let guard = self.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + unsafe { bindings::gpu_buddy_fini(guard.as_raw()) }; + } +} + +// SAFETY: `GpuBuddyInner` can be sent between threads. +unsafe impl Send for GpuBuddyInner {} + +// SAFETY: `GpuBuddyInner` is `Sync` because `GpuBuddyInner::lock` +// serializes all access to the C allocator, preventing data races. +unsafe impl Sync for GpuBuddyInner {} + +/// Guard that proves the lock is held, enabling access to the allocator. +/// +/// The `_guard` holds the lock for the duration of this guard's lifetime. +struct GpuBuddyGuard<'a> { + inner: &'a GpuBuddyInner, + _guard: MutexGuard<'a, ()>, +} + +impl GpuBuddyGuard<'_> { + /// Get a raw pointer to the underlying C `gpu_buddy` structure. + fn as_raw(&self) -> *mut bindings::gpu_buddy { + self.inner.inner.get() + } +} + +/// GPU buddy allocator instance. +/// +/// This structure wraps the C `gpu_buddy` allocator using reference counting. +/// The allocator is automatically cleaned up when all references are dropped. +/// +/// Refer to the module-level documentation for usage examples. +pub struct GpuBuddy(Arc); + +impl GpuBuddy { + /// Create a new buddy allocator. + /// + /// The allocator manages a contiguous address space of the given size, with the + /// specified minimum allocation unit (chunk_size must be at least 4KB). + pub fn new(params: GpuBuddyParams) -> Result { + Arc::pin_init(GpuBuddyInner::new(params), GFP_KERNEL).map(Self) + } + + /// Get the base offset for allocations. + pub fn base_offset(&self) -> u64 { + self.0.params.base_offset + } + + /// Get the chunk size (minimum allocation unit). + pub fn chunk_size(&self) -> Alignment { + self.0.params.chunk_size + } + + /// Get the total managed size. + pub fn size(&self) -> u64 { + self.0.params.size + } + + /// Get the available (free) memory in bytes. + pub fn avail(&self) -> u64 { + let guard = self.0.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized allocator. + // `guard` provides exclusive access. + unsafe { (*guard.as_raw()).avail } + } + + /// Allocate blocks from the buddy allocator. + /// + /// Returns a pin-initializer for [`AllocatedBlocks`]. + pub fn alloc_blocks( + &self, + mode: GpuBuddyAllocMode, + size: u64, + min_block_size: Alignment, + flags: impl Into, + ) -> impl PinInit { + let buddy_arc = Arc::clone(&self.0); + let (start, end) = mode.range(); + let mode_flags = mode.as_flags(); + let modifier_flags = flags.into(); + + // Create pin-initializer that initializes list and allocates blocks. + try_pin_init!(AllocatedBlocks { + buddy: buddy_arc, + list <- CListHead::new(), + _: { + // Reject zero-sized or inverted ranges. + if let GpuBuddyAllocMode::Range(range) = &mode { + if range.is_empty() { + Err::<(), Error>(EINVAL)?; + } + } + + // Lock while allocating to serialize with concurrent frees. + let guard = buddy.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + to_result(unsafe { + bindings::gpu_buddy_alloc_blocks( + guard.as_raw(), + start, + end, + size, + min_block_size.as_usize() as u64, + list.as_raw(), + mode_flags | usize::from(modifier_flags), + ) + })? + } + }) + } +} + +/// Allocated blocks from the buddy allocator with automatic cleanup. +/// +/// This structure owns a list of allocated blocks and ensures they are +/// automatically freed when dropped. Use `iter()` to iterate over all +/// allocated blocks. +/// +/// # Invariants +/// +/// - `list` is an initialized, valid list head containing allocated blocks. +#[pin_data(PinnedDrop)] +pub struct AllocatedBlocks { + #[pin] + list: CListHead, + buddy: Arc, +} + +impl AllocatedBlocks { + /// Check if the block list is empty. + pub fn is_empty(&self) -> bool { + // An empty list head points to itself. + !self.list.is_linked() + } + + /// Iterate over allocated blocks. + /// + /// Returns an iterator yielding [`AllocatedBlock`] values. Each [`AllocatedBlock`] + /// borrows `self` and is only valid for the duration of that borrow. + pub fn iter(&self) -> impl Iterator> + '_ { + let head = self.list.as_raw(); + // SAFETY: Per the type invariant, `list` is an initialized sentinel `list_head` + // and is not concurrently modified (we hold a `&self` borrow). The list contains + // `gpu_buddy_block` items linked via `__bindgen_anon_1.link`. `Block` is + // `#[repr(transparent)]` over `gpu_buddy_block`. + let clist = unsafe { + clist_create!( + head, + Block, + bindings::gpu_buddy_block, + __bindgen_anon_1.link + ) + }; + + clist + .iter() + .map(|this| AllocatedBlock { this, blocks: self }) + } +} + +#[pinned_drop] +impl PinnedDrop for AllocatedBlocks { + fn drop(self: Pin<&mut Self>) { + let guard = self.buddy.lock(); + + // SAFETY: + // - list is valid per the type's invariants. + // - guard provides exclusive access to the allocator. + unsafe { + bindings::gpu_buddy_free_list(guard.as_raw(), self.list.as_raw(), 0); + } + } +} + +/// A GPU buddy block. +/// +/// Transparent wrapper over C `gpu_buddy_block` structure. This type is returned +/// as references during iteration over [`AllocatedBlocks`]. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains a valid, allocated `gpu_buddy_block`. +#[repr(transparent)] +struct Block(Opaque); + +impl Block { + /// Get a raw pointer to the underlying C block. + fn as_raw(&self) -> *mut bindings::gpu_buddy_block { + self.0.get() + } + + /// Get the block's raw offset in the buddy address space (without base offset). + fn offset(&self) -> u64 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_offset(self.as_raw()) } + } + + /// Get the block order. + fn order(&self) -> u32 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_order(self.as_raw()) } + } +} + +// SAFETY: `Block` is a wrapper around `gpu_buddy_block` which can be +// sent across threads safely. +unsafe impl Send for Block {} + +// SAFETY: `Block` is only accessed through shared references after +// allocation, and thus safe to access concurrently across threads. +unsafe impl Sync for Block {} + +/// A buddy block paired with its owning [`AllocatedBlocks`] context. +/// +/// Unlike a raw block, which only knows its offset within the buddy address +/// space, an [`AllocatedBlock`] also has access to the allocator's `base_offset` +/// and `chunk_size`, enabling it to compute absolute offsets and byte sizes. +/// +/// Returned by [`AllocatedBlocks::iter()`]. +pub struct AllocatedBlock<'a> { + this: &'a Block, + blocks: &'a AllocatedBlocks, +} + +impl AllocatedBlock<'_> { + /// Get the block's offset in the address space. + /// + /// Returns the absolute offset including the allocator's base offset. + /// This is the actual address to use for accessing the allocated memory. + pub fn offset(&self) -> u64 { + self.blocks.buddy.params.base_offset + self.this.offset() + } + + /// Get the block order (size = chunk_size << order). + pub fn order(&self) -> u32 { + self.this.order() + } + + /// Get the block's size in bytes. + pub fn size(&self) -> u64 { + (self.blocks.buddy.params.chunk_size.as_usize() as u64) << self.this.order() + } +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs new file mode 100644 index 000000000..0487bae81 --- /dev/null +++ b/rust/kernel/i2c.rs @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! I2C Driver subsystem + +// I2C Driver abstractions. +use crate::{ + acpi, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + devres::Devres, + driver, + error::*, + of, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::{ + from_ref, + NonNull, // + }, // +}; + +/// An I2C device id table. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::i2c_device_id); + +impl DeviceId { + const I2C_NAME_SIZE: usize = 20; + + /// Create a new device id from an I2C 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_NAME_SIZE, "ID exceeds 20 bytes"); + let mut i2c: bindings::i2c_device_id = pin_init::zeroed(); + let mut i = 0; + while i < src.len() { + i2c.name[i] = src[i]; + i += 1; + } + + Self(i2c) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `i2c_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::i2c_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); +} + +/// IdTable type for I2C +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a I2C `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! i2c_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("i2c", $crate::i2c::DeviceId, $($tt)*); + }; +} + +/// An adapter for the registration of I2C drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::i2c_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct i2c_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::i2c_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + idrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + build_assert!( + T::ACPI_ID_TABLE.is_some() || T::OF_ID_TABLE.is_some() || T::I2C_ID_TABLE.is_some(), + "At least one of ACPI/OF/Legacy tables must be present when registering an i2c driver" + ); + + let i2c_table = match T::I2C_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct i2c_client` on initialization. + unsafe { + (*idrv.get()).driver.name = name.as_char_ptr(); + (*idrv.get()).probe = Some(Self::probe_callback); + (*idrv.get()).remove = Some(Self::remove_callback); + (*idrv.get()).shutdown = Some(Self::shutdown_callback); + (*idrv.get()).id_table = i2c_table; + (*idrv.get()).driver.of_match_table = of_table; + (*idrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::i2c_register_driver(module.as_ptr(), idrv.get()) }) + } + + unsafe fn unregister(idrv: &Opaque) { + // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::i2c_del_driver(idrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_int { + // SAFETY: The I2C bus only ever calls the probe callback with a valid pointer to a + // `struct i2c_client`. + // + // INVARIANT: `idev` is valid for the duration of `probe_callback()`. + let idev = unsafe { &*idev.cast::>>() }; + + let info = Self::i2c_id_info(idev).or_else(|| { + // SAFETY: `idev` matched data is of type `Self::IdInfo`. + unsafe { ::id_info(idev.as_ref()) } + }); + + from_result(|| { + let data = T::probe(idev, info); + + idev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + let idev = unsafe { &*idev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; + + T::unbind(idev, data); + } + + extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `shutdown_callback` is only ever called for a valid `idev` + let idev = unsafe { &*idev.cast::>>() }; + + // SAFETY: `shutdown_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; + + T::shutdown(idev, data); + } + + /// The [`i2c::IdTable`] of the corresponding driver. + fn i2c_id_table() -> Option::IdInfo>> { + T::I2C_ID_TABLE + } + + /// Returns the driver's private data from the matching entry in the [`i2c::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`i2c::IdTable`]. + fn i2c_id_info(dev: &I2cClient) -> Option<&'static ::IdInfo> { + let table = Self::i2c_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for reads + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::i2c_match_id(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + return None; + } + + // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct i2c_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::() }; + + // SAFETY: `id` comes from `table` which is of type `IdArray<_, Self::IdInfo>`. + Some(unsafe { id.info_unchecked::() }) + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single i2c driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_i2c_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_i2c_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::i2c::Adapter, { $($f)* }); + }; +} + +/// The i2c driver trait. +/// +/// Drivers must implement this trait in order to get a i2c driver registered. +/// +/// # Example +/// +///``` +/// # use kernel::{acpi, bindings, device::Core, i2c, of}; +/// +/// struct MyDriver; +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// kernel::i2c_device_table!( +/// I2C_TABLE, +/// ::IdInfo, +/// [ +/// (i2c::DeviceId::new(c"rust_driver_i2c"), ()) +/// ] +/// ); +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// impl i2c::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// +/// fn shutdown<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// this: Pin<&Self::Data<'bound>>, +/// ) { +/// } +/// } +///``` +pub trait Driver { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const I2C_ID_TABLE: Option> = None; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// I2C driver probe. + /// + /// Called when a new i2c client is added or discovered. + /// Implementers should attempt to initialize the client here. + fn probe<'bound>( + dev: &'bound I2cClient>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// I2C driver shutdown. + /// + /// Called by the kernel during system reboot or power-off to allow the [`Driver`] to bring the + /// [`I2cClient`] into a safe state. Implementing this callback is optional. + /// + /// Typical actions include stopping transfers, disabling interrupts, or resetting the hardware + /// to prevent undesired behavior during shutdown. + /// + /// This callback is distinct from final resource cleanup, as the driver instance remains valid + /// after it returns. Any deallocation or teardown of driver-owned resources should instead be + /// handled in `Drop`. + fn shutdown<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } + + /// I2C driver unbind. + /// + /// Called when the [`I2cClient`] is unbound from its bound [`Driver`]. Implementing this + /// callback is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The i2c adapter representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_adapter`. The +/// implementation abstracts the usage of an existing C `struct i2c_adapter` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cAdapter`] instance represents a valid `struct i2c_adapter` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cAdapter( + Opaque, + PhantomData, +); + +impl I2cAdapter { + fn as_raw(&self) -> *mut bindings::i2c_adapter { + self.0.get() + } +} + +impl I2cAdapter { + /// Returns the I2C Adapter index. + #[inline] + pub fn index(&self) -> i32 { + // SAFETY: `self.as_raw` is a valid pointer to a `struct i2c_adapter`. + unsafe { (*self.as_raw()).nr } + } + + /// Gets pointer to an `i2c_adapter` by index. + #[inline] + pub fn get(index: i32) -> Result> { + // SAFETY: `index` must refer to a valid I2C adapter; the kernel + // guarantees that `i2c_get_adapter(index)` returns either a valid + // pointer or NULL. `NonNull::new` guarantees the correct check. + let adapter = NonNull::new(unsafe { bindings::i2c_get_adapter(index) }).ok_or(ENODEV)?; + + // SAFETY: `adapter` is non-null and points to a live `i2c_adapter`. + // `I2cAdapter` is #[repr(transparent)], so this cast is valid. + // `i2c_get_adapter` returned the adapter with an incremented refcount, which we pass to + // the `ARef`. + Ok(unsafe { ARef::from_raw(adapter.cast::>()) }) + } +} + +// SAFETY: `I2cAdapter` is a transparent wrapper of a type that doesn't depend on +// `I2cAdapter`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cAdapter }); +kernel::impl_device_context_into_aref!(I2cAdapter); + +// SAFETY: Instances of `I2cAdapter` are always reference-counted. +unsafe impl AlwaysRefCounted for I2cAdapter { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::i2c_get_adapter(self.index()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } + } +} + +/// The i2c board info representation +/// +/// This structure represents the Rust abstraction for a C `struct i2c_board_info` structure, +/// which is used for manual I2C client creation. +#[repr(transparent)] +pub struct I2cBoardInfo(bindings::i2c_board_info); + +impl I2cBoardInfo { + const I2C_TYPE_SIZE: usize = 20; + /// Create a new [`I2cBoardInfo`] for a kernel driver. + #[inline(always)] + pub const fn new(type_: &'static CStr, addr: u16) -> Self { + let src = type_.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_TYPE_SIZE, "Type exceeds 20 bytes"); + let mut i2c_board_info: bindings::i2c_board_info = pin_init::zeroed(); + let mut i: usize = 0; + while i < src.len() { + i2c_board_info.type_[i] = src[i]; + i += 1; + } + + i2c_board_info.addr = addr; + Self(i2c_board_info) + } + + fn as_raw(&self) -> *const bindings::i2c_board_info { + from_ref(&self.0) + } +} + +/// The i2c client representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_client`. The +/// implementation abstracts the usage of an existing C `struct i2c_client` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cClient`] instance represents a valid `struct i2c_client` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cClient( + Opaque, + PhantomData, +); + +impl I2cClient { + fn as_raw(&self) -> *mut bindings::i2c_client { + self.0.get() + } +} + +// SAFETY: `I2cClient` is a transparent wrapper of `struct i2c_client`. +// The offset is guaranteed to point to a valid device field inside `I2cClient`. +unsafe impl device::AsBusDevice for I2cClient { + const OFFSET: usize = offset_of!(bindings::i2c_client, dev); +} + +// SAFETY: `I2cClient` is a transparent wrapper of a type that doesn't depend on +// `I2cClient`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cClient }); +kernel::impl_device_context_into_aref!(I2cClient); + +// SAFETY: Instances of `I2cClient` are always reference-counted. +unsafe impl AlwaysRefCounted for I2cClient { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(&raw mut (*obj.as_ref().as_raw()).dev) } + } +} + +impl AsRef> for I2cClient { + fn as_ref(&self) -> &device::Device { + let raw = self.as_raw(); + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct i2c_client`. + let dev = unsafe { &raw mut (*raw).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &I2cClient { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if unsafe { bindings::i2c_verify_client(dev.as_raw()).is_null() } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the type of `dev` equals to + // `bindings::i2c_client_type`, hence `dev` must be embedded in a valid + // `struct i2c_client` as guaranteed by the corresponding C code. + let idev = unsafe { container_of!(dev.as_raw(), bindings::i2c_client, dev) }; + + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + Ok(unsafe { &*idev.cast() }) + } +} + +// SAFETY: A `I2cClient` is always reference-counted and can be released from any thread. +unsafe impl Send for I2cClient {} + +// SAFETY: `I2cClient` can be shared among threads because all methods of `I2cClient` +// (i.e. `I2cClient) are thread safe. +unsafe impl Sync for I2cClient {} + +/// The registration of an i2c client device. +/// +/// This type represents the registration of a [`struct i2c_client`]. When an instance of this +/// type is dropped, its respective i2c client device will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.0` always holds a valid pointer to an initialized and registered +/// [`struct i2c_client`]. +#[repr(transparent)] +pub struct Registration(NonNull); + +impl Registration { + /// The C `i2c_new_client_device` function wrapper for manual I2C client creation. + pub fn new<'a>( + i2c_adapter: &I2cAdapter, + i2c_board_info: &I2cBoardInfo, + parent_dev: &'a device::Device, + ) -> impl PinInit, Error> + 'a { + Devres::new(parent_dev, Self::try_new(i2c_adapter, i2c_board_info)) + } + + fn try_new(i2c_adapter: &I2cAdapter, i2c_board_info: &I2cBoardInfo) -> Result { + // SAFETY: the kernel guarantees that `i2c_new_client_device()` returns either a valid + // pointer or NULL. `from_err_ptr` separates errors. Following `NonNull::new` + // checks for NULL. + let raw_dev = from_err_ptr(unsafe { + bindings::i2c_new_client_device(i2c_adapter.as_raw(), i2c_board_info.as_raw()) + })?; + + let dev_ptr = NonNull::new(raw_dev).ok_or(ENODEV)?; + + Ok(Self(dev_ptr)) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `Drop` is only called for a valid `Registration`, which by invariant + // always contains a non-null pointer to an `i2c_client`. + unsafe { bindings::i2c_unregister_device(self.0.as_ptr()) } + } +} + +// SAFETY: A `Registration` of a `struct i2c_client` can be released from any thread. +unsafe impl Send for Registration {} + +// SAFETY: `Registration` offers no interior mutability (no mutation through &self +// and no mutable access is exposed) +unsafe impl Sync for Registration {} diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs new file mode 100644 index 000000000..384753fe0 --- /dev/null +++ b/rust/kernel/id_pool.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! Rust API for an ID pool backed by a [`BitmapVec`]. + +use crate::alloc::{AllocError, Flags}; +use crate::bitmap::BitmapVec; + +/// Represents a dynamic ID pool backed by a [`BitmapVec`]. +/// +/// Clients acquire and release IDs from unset bits in a bitmap. +/// +/// The capacity of the ID pool may be adjusted by users as +/// needed. The API supports the scenario where users need precise control +/// over the time of allocation of a new backing bitmap, which may require +/// release of spinlock. +/// Due to concurrent updates, all operations are re-verified to determine +/// if the grow or shrink is sill valid. +/// +/// # Examples +/// +/// Basic usage +/// +/// ``` +/// use kernel::alloc::AllocError; +/// use kernel::id_pool::{IdPool, UnusedId}; +/// +/// let mut pool = IdPool::with_capacity(64, GFP_KERNEL)?; +/// for i in 0..64 { +/// assert_eq!(i, pool.find_unused_id(i).ok_or(ENOSPC)?.acquire()); +/// } +/// +/// pool.release_id(23); +/// assert_eq!(23, pool.find_unused_id(0).ok_or(ENOSPC)?.acquire()); +/// +/// assert!(pool.find_unused_id(0).is_none()); // time to realloc. +/// let resizer = pool.grow_request().ok_or(ENOSPC)?.realloc(GFP_KERNEL)?; +/// pool.grow(resizer); +/// +/// assert_eq!(pool.find_unused_id(0).ok_or(ENOSPC)?.acquire(), 64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Releasing spinlock to grow the pool +/// +/// ```no_run +/// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; +/// use kernel::sync::{new_spinlock, SpinLock}; +/// use kernel::id_pool::IdPool; +/// +/// fn get_id_maybe_realloc(guarded_pool: &SpinLock) -> Result { +/// let mut pool = guarded_pool.lock(); +/// loop { +/// match pool.find_unused_id(0) { +/// Some(index) => return Ok(index.acquire()), +/// None => { +/// let alloc_request = pool.grow_request(); +/// drop(pool); +/// let resizer = alloc_request.ok_or(AllocError)?.realloc(GFP_KERNEL)?; +/// pool = guarded_pool.lock(); +/// pool.grow(resizer) +/// } +/// } +/// } +/// } +/// ``` +pub struct IdPool { + map: BitmapVec, +} + +/// Indicates that an [`IdPool`] should change to a new target size. +pub struct ReallocRequest { + num_ids: usize, +} + +/// Contains a [`BitmapVec`] of a size suitable for reallocating [`IdPool`]. +pub struct PoolResizer { + new: BitmapVec, +} + +impl ReallocRequest { + /// Allocates a new backing [`BitmapVec`] for [`IdPool`]. + /// + /// This method only prepares reallocation and does not complete it. + /// Reallocation will complete after passing the [`PoolResizer`] to the + /// [`IdPool::grow`] or [`IdPool::shrink`] operation, which will check + /// that reallocation still makes sense. + pub fn realloc(&self, flags: Flags) -> Result { + let new = BitmapVec::new(self.num_ids, flags)?; + Ok(PoolResizer { new }) + } +} + +impl IdPool { + /// Constructs a new [`IdPool`]. + /// + /// The pool will have a capacity of [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn new() -> Self { + Self { + map: BitmapVec::new_inline(), + } + } + + /// Constructs a new [`IdPool`] with space for a specific number of bits. + /// + /// A capacity below [`MAX_INLINE_LEN`] is adjusted to [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn with_capacity(num_ids: usize, flags: Flags) -> Result { + let num_ids = usize::max(num_ids, BitmapVec::MAX_INLINE_LEN); + let map = BitmapVec::new(num_ids, flags)?; + Ok(Self { map }) + } + + /// Returns how many IDs this pool can currently have. + #[inline] + pub fn capacity(&self) -> usize { + self.map.len() + } + + /// Returns a [`ReallocRequest`] if the [`IdPool`] can be shrunk, [`None`] otherwise. + /// + /// The capacity of an [`IdPool`] cannot be shrunk below [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// alloc::AllocError, + /// bitmap::BitmapVec, + /// id_pool::{ + /// IdPool, + /// ReallocRequest, + /// }, + /// }; + /// + /// let mut pool = IdPool::with_capacity(1024, GFP_KERNEL)?; + /// let alloc_request = pool.shrink_request().ok_or(AllocError)?; + /// let resizer = alloc_request.realloc(GFP_KERNEL)?; + /// pool.shrink(resizer); + /// assert_eq!(pool.capacity(), BitmapVec::MAX_INLINE_LEN); + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn shrink_request(&self) -> Option { + let cap = self.capacity(); + // Shrinking below `MAX_INLINE_LEN` is never possible. + if cap <= BitmapVec::MAX_INLINE_LEN { + return None; + } + // Determine if the bitmap can shrink based on the position of + // its last set bit. If the bit is within the first quarter of + // the bitmap then shrinking is possible. In this case, the + // bitmap should shrink to half its current size. + let Some(bit) = self.map.last_bit() else { + return Some(ReallocRequest { + num_ids: BitmapVec::MAX_INLINE_LEN, + }); + }; + if bit >= (cap / 4) { + return None; + } + let num_ids = usize::max(BitmapVec::MAX_INLINE_LEN, cap / 2); + Some(ReallocRequest { num_ids }) + } + + /// Shrinks pool by using a new [`BitmapVec`], if still possible. + #[inline] + pub fn shrink(&mut self, mut resizer: PoolResizer) { + // Between request to shrink that led to allocation of `resizer` and now, + // bits may have changed. + // Verify that shrinking is still possible. In case shrinking to + // the size of `resizer` is no longer possible, do nothing, + // drop `resizer` and move on. + let Some(updated) = self.shrink_request() else { + return; + }; + if updated.num_ids > resizer.new.len() { + return; + } + + resizer.new.copy_and_extend(&self.map); + self.map = resizer.new; + } + + /// Returns a [`ReallocRequest`] for growing this [`IdPool`], if possible. + /// + /// The capacity of an [`IdPool`] cannot be grown above [`MAX_LEN`]. + /// + /// [`MAX_LEN`]: BitmapVec::MAX_LEN + #[inline] + pub fn grow_request(&self) -> Option { + let num_ids = self.capacity() * 2; + if num_ids > BitmapVec::MAX_LEN { + return None; + } + Some(ReallocRequest { num_ids }) + } + + /// Grows pool by using a new [`BitmapVec`], if still necessary. + /// + /// The `resizer` arguments has to be obtained by calling [`Self::grow_request`] + /// on this object and performing a [`ReallocRequest::realloc`]. + #[inline] + pub fn grow(&mut self, mut resizer: PoolResizer) { + // Between request to grow that led to allocation of `resizer` and now, + // another thread may have already grown the capacity. + // In this case, do nothing, drop `resizer` and move on. + if resizer.new.len() <= self.capacity() { + return; + } + + resizer.new.copy_and_extend(&self.map); + self.map = resizer.new; + } + + /// Finds an unused ID in the bitmap. + /// + /// Upon success, returns its index. Otherwise, returns [`None`] + /// to indicate that a [`Self::grow_request`] is needed. + #[inline] + #[must_use] + pub fn find_unused_id(&mut self, offset: usize) -> Option> { + // INVARIANT: `next_zero_bit()` returns None or an integer less than `map.len()` + Some(UnusedId { + id: self.map.next_zero_bit(offset)?, + pool: self, + }) + } + + /// Releases an ID. + #[inline] + pub fn release_id(&mut self, id: usize) { + self.map.clear_bit(id); + } +} + +/// Represents an unused id in an [`IdPool`]. +/// +/// # Invariants +/// +/// The value of `id` is less than `pool.map.len()`. +pub struct UnusedId<'pool> { + id: usize, + pool: &'pool mut IdPool, +} + +impl<'pool> UnusedId<'pool> { + /// Get the unused id as an usize. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_usize(&self) -> usize { + self.id + } + + /// Get the unused id as an u32. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_u32(&self) -> u32 { + // CAST: By the type invariants: + // `self.id < pool.map.len() <= BitmapVec::MAX_LEN = i32::MAX`. + self.id as u32 + } + + /// Acquire the unused id. + #[inline] + pub fn acquire(self) -> usize { + self.pool.map.set_bit(self.id); + self.id + } +} + +impl Default for IdPool { + #[inline] + fn default() -> Self { + Self::new() + } +} diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs new file mode 100644 index 000000000..fdf44d5ee --- /dev/null +++ b/rust/kernel/impl_flags.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bitflag type generator. + +/// Common helper for declaring bitflag and bitmask types. +/// +/// This macro takes as input: +/// - A struct declaration representing a bitmask type +/// (e.g., `pub struct Permissions(u32)`). +/// - An enumeration declaration representing individual bit flags +/// (e.g., `pub enum Permission { ... }`). +/// +/// And generates: +/// - The struct and enum types with appropriate `#[repr]` attributes. +/// - Implementations of common bitflag operators +/// ([`::core::ops::BitOr`], [`::core::ops::BitAnd`], etc.). +/// - Utility methods such as `.contains()` to check flags. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// bits::bit_u32, +/// impl_flags, // +/// }; +/// +/// impl_flags!( +/// /// Represents multiple permissions. +/// #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] +/// pub struct Permissions(u32); +/// +/// /// Represents a single permission. +/// #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// pub enum Permission { +/// /// Read permission. +/// Read = bit_u32(0), +/// +/// /// Write permission. +/// Write = bit_u32(1), +/// +/// /// Execute permission. +/// Execute = bit_u32(2), +/// } +/// ); +/// +/// // Combine multiple permissions using the bitwise OR (`|`) operator. +/// let mut read_write: Permissions = Permission::Read | Permission::Write; +/// assert!(read_write.contains(Permission::Read)); +/// assert!(read_write.contains(Permission::Write)); +/// assert!(!read_write.contains(Permission::Execute)); +/// assert!(read_write.contains_any(Permission::Read | Permission::Execute)); +/// assert!(read_write.contains_all(Permission::Read | Permission::Write)); +/// +/// // Using the bitwise OR assignment (`|=`) operator. +/// read_write |= Permission::Execute; +/// assert!(read_write.contains(Permission::Execute)); +/// +/// // Masking a permission with the bitwise AND (`&`) operator. +/// let read_only: Permissions = read_write & Permission::Read; +/// assert!(read_only.contains(Permission::Read)); +/// assert!(!read_only.contains(Permission::Write)); +/// +/// // Toggling permissions with the bitwise XOR (`^`) operator. +/// let toggled: Permissions = read_only ^ Permission::Read; +/// assert!(!toggled.contains(Permission::Read)); +/// +/// // Inverting permissions with the bitwise NOT (`!`) operator. +/// let negated = !read_only; +/// assert!(negated.contains(Permission::Write)); +/// assert!(!negated.contains(Permission::Read)); +/// ``` +#[macro_export] +macro_rules! impl_flags { + ( + $(#[$outer_flags:meta])* + $vis_flags:vis struct $flags:ident($ty:ty); + + $(#[$outer_flag:meta])* + $vis_flag:vis enum $flag:ident { + $( + $(#[$inner_flag:meta])* + $name:ident = $value:expr + ),+ $( , )? + } + ) => { + $(#[$outer_flags])* + #[repr(transparent)] + $vis_flags struct $flags($ty); + + $(#[$outer_flag])* + #[repr($ty)] + $vis_flag enum $flag { + $( + $(#[$inner_flag])* + $name = $value + ),+ + } + + impl ::core::convert::From<$flag> for $flags { + #[inline] + fn from(value: $flag) -> Self { + Self(value as $ty) + } + } + + impl ::core::convert::From<$flags> for $ty { + #[inline] + fn from(value: $flags) -> Self { + value.0 + } + } + + impl ::core::ops::BitOr for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl ::core::ops::BitOrAssign for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitOr<$flag> for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: $flag) -> Self::Output { + self | Self::from(rhs) + } + } + + impl ::core::ops::BitOrAssign<$flag> for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: $flag) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitAnd for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl ::core::ops::BitAndAssign for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitAnd<$flag> for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: $flag) -> Self::Output { + self & Self::from(rhs) + } + } + + impl ::core::ops::BitAndAssign<$flag> for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: $flag) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitXor for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + Self((self.0 ^ rhs.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitXorAssign for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::BitXor<$flag> for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: $flag) -> Self::Output { + self ^ Self::from(rhs) + } + } + + impl ::core::ops::BitXorAssign<$flag> for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: $flag) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::Not for $flags { + type Output = Self; + #[inline] + fn not(self) -> Self::Output { + Self((!self.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitOr for $flag { + type Output = $flags; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + $flags(self as $ty | rhs as $ty) + } + } + + impl ::core::ops::BitAnd for $flag { + type Output = $flags; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + $flags(self as $ty & rhs as $ty) + } + } + + impl ::core::ops::BitXor for $flag { + type Output = $flags; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + $flags((self as $ty ^ rhs as $ty) & $flags::all_bits()) + } + } + + impl ::core::ops::Not for $flag { + type Output = $flags; + #[inline] + fn not(self) -> Self::Output { + $flags((!(self as $ty)) & $flags::all_bits()) + } + } + + impl $flags { + /// Returns an empty instance where no flags are set. + #[inline] + pub const fn empty() -> Self { + Self(0) + } + + /// Returns a mask containing all valid flag bits. + #[inline] + pub const fn all_bits() -> $ty { + 0 $( | $value )+ + } + + /// Checks if a specific flag is set. + #[inline] + pub fn contains(self, flag: $flag) -> bool { + (self.0 & flag as $ty) == flag as $ty + } + + /// Checks if at least one of the provided flags is set. + #[inline] + pub fn contains_any(self, flags: $flags) -> bool { + (self.0 & flags.0) != 0 + } + + /// Checks if all of the provided flags are set. + #[inline] + pub fn contains_all(self, flags: $flags) -> bool { + (self.0 & flags.0) == flags.0 + } + } + }; +} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs new file mode 100644 index 000000000..1fdc3963e --- /dev/null +++ b/rust/kernel/init.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to the [`pin-init`] crate. +//! +//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential +//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! The [`pin-init`] crate is the way such structs are initialized on the Rust side. Please refer +//! to its documentation to better understand how to use it. Additionally, there are many examples +//! throughout the kernel, such as the types from the [`sync`] module. And the ones presented +//! below. +//! +//! [`sync`]: crate::sync +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [`pin-init`]: https://rust.docs.kernel.org/pin_init/ +//! +//! # [`Opaque`] +//! +//! For the special case where initializing a field is a single FFI-function call that cannot fail, +//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single +//! [`Opaque`] field by just delegating to the supplied closure. You can use these in +//! combination with [`pin_init!`]. +//! +//! [`Opaque`]: crate::types::Opaque +//! [`Opaque::ffi_init`]: crate::types::Opaque::ffi_init +//! [`pin_init!`]: pin_init::pin_init +//! +//! # Examples +//! +//! ## General Examples +//! +//! ```rust +//! # #![expect(clippy::undocumented_unsafe_blocks)] +//! use kernel::types::Opaque; +//! use pin_init::pin_init_from_closure; +//! +//! // assume we have some `raw_foo` type in C: +//! #[repr(C)] +//! struct RawFoo([u8; 16]); +//! extern "C" { +//! fn init_foo(_: *mut RawFoo); +//! } +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! raw: Opaque, +//! } +//! +//! impl Foo { +//! fn setup(self: Pin<&mut Self>) { +//! pr_info!("Setting up foo\n"); +//! } +//! } +//! +//! let foo = pin_init!(Foo { +//! raw <- unsafe { +//! Opaque::ffi_init(|s| { +//! // note that this cannot fail. +//! init_foo(s); +//! }) +//! }, +//! }).pin_chain(|foo| { +//! foo.setup(); +//! Ok(()) +//! }); +//! ``` +//! +//! ```rust +//! use kernel::{prelude::*, types::Opaque}; +//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; +//! # mod bindings { +//! # #![expect(non_camel_case_types, clippy::missing_safety_doc)] +//! # pub struct foo; +//! # pub unsafe fn init_foo(_ptr: *mut foo) {} +//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} +//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } +//! # } +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! foo: Opaque, +//! #[pin] +//! _p: PhantomPinned, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init::pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = addr_of_mut!((*slot).foo); +//! +//! // Initialize the `foo` +//! bindings::init_foo(Opaque::cast_into(foo)); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(Opaque::cast_into(foo), flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(Opaque::cast_into(foo)); +//! return Err(Error::from_errno(err)); +//! } +//! +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get()) }; +//! } +//! } +//! ``` + +use crate::{ + alloc::{AllocError, Flags}, + error::{self, Error}, +}; +use pin_init::{init_from_closure, pin_init_from_closure, Init, PinInit}; + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit: Sized { + /// Pinned version of `Self`. + /// + /// If a type already implicitly pins its pointee, `Pin` is unnecessary. In this case use + /// `Self`, otherwise just use `Pin`. + type PinnedSelf; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] + fn pin_init(init: impl PinInit, flags: Flags) -> error::Result + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| { + pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)) + }) + }; + Self::try_pin_init(init, flags) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From; + + /// Use the given initializer to in-place initialize a `T`. + #[inline] + fn init(init: impl Init, flags: Flags) -> error::Result + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))) + }; + Self::try_init(init, flags) + } +} + +/// Construct an in-place fallible initializer for `struct`s. +/// +/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use +/// [`init!`]. +/// +/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, +/// append `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::error::Error; +/// use pin_init::init_zeroed; +/// struct BigBuf { +/// big: KBox<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init { +/// try_init!(Self { +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, +/// small: [0; 1024 * 1024], +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Infallible`]: core::convert::Infallible +/// [`init!`]: pin_init::init +/// [`try_pin_init!`]: crate::try_pin_init! +/// [`Error`]: crate::error::Error +#[macro_export] +macro_rules! try_init { + ($($args:tt)*) => { + ::pin_init::init!( + #[default_error($crate::error::Error)] + $($args)* + ) + } +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. +/// +/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop +/// initialization and return the error. +/// +/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when +/// initialization fails, the memory can be safely deallocated without any further modifications. +/// +/// This macro defaults the error to [`Error`]. +/// +/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` +/// after the `struct` initializer to specify the error type you want to use. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(new_uninit)] +/// use kernel::error::Error; +/// use pin_init::init_zeroed; +/// #[pin_data] +/// struct BigBuf { +/// big: KBox<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// ptr: *mut u8, +/// } +/// +/// impl BigBuf { +/// fn new() -> impl PinInit { +/// try_pin_init!(Self { +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, +/// small: [0; 1024 * 1024], +/// ptr: core::ptr::null_mut(), +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Infallible`]: core::convert::Infallible +/// [`pin_init!`]: pin_init::pin_init +/// [`Error`]: crate::error::Error +#[macro_export] +macro_rules! try_pin_init { + ($($args:tt)*) => { + ::pin_init::pin_init!( + #[default_error($crate::error::Error)] + $($args)* + ) + } +} diff --git a/rust/kernel/interop.rs b/rust/kernel/interop.rs new file mode 100644 index 000000000..3b371d782 --- /dev/null +++ b/rust/kernel/interop.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for interfacing Rust code with C kernel subsystems. +//! +//! This module is intended for low-level, unsafe Rust infrastructure code +//! that interoperates between Rust and C. It is *not* for use directly in +//! Rust drivers. + +pub mod list; diff --git a/rust/kernel/interop/list.rs b/rust/kernel/interop/list.rs new file mode 100644 index 000000000..54265ea03 --- /dev/null +++ b/rust/kernel/interop/list.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust interface for C doubly circular intrusive linked lists. +//! +//! This module provides Rust abstractions for iterating over C `list_head`-based +//! linked lists. It should only be used for cases where C and Rust code share +//! direct access to the same linked list through a C interop interface. +//! +//! Note: This *must not* be used by Rust components that just need a linked list +//! primitive. Use [`kernel::list::List`] instead. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! bindings, +//! interop::list::clist_create, +//! types::Opaque, +//! }; +//! # // Create test list with values (0, 10, 20) - normally done by C code but it is +//! # // emulated here for doctests using the C bindings. +//! # use core::mem::MaybeUninit; +//! # +//! # /// C struct with embedded `list_head` (typically will be allocated by C code). +//! # #[repr(C)] +//! # pub struct SampleItemC { +//! # pub value: i32, +//! # pub link: bindings::list_head, +//! # } +//! # +//! # let mut head = MaybeUninit::::uninit(); +//! # +//! # let head = head.as_mut_ptr(); +//! # // SAFETY: `head` and all the items are test objects allocated in this scope. +//! # unsafe { bindings::INIT_LIST_HEAD(head) }; +//! # +//! # let mut items = [ +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # ]; +//! # +//! # for (i, item) in items.iter_mut().enumerate() { +//! # let ptr = item.as_mut_ptr(); +//! # // SAFETY: `ptr` points to a valid `MaybeUninit`. +//! # unsafe { (*ptr).value = i as i32 * 10 }; +//! # // SAFETY: `&raw mut` creates a pointer valid for `INIT_LIST_HEAD`. +//! # unsafe { bindings::INIT_LIST_HEAD(&raw mut (*ptr).link) }; +//! # // SAFETY: `link` was just initialized and `head` is a valid list head. +//! # unsafe { bindings::list_add_tail(&mut (*ptr).link, head) }; +//! # } +//! +//! /// Rust wrapper for the C struct. +//! /// +//! /// The list item struct in this example is defined in C code as: +//! /// +//! /// ```c +//! /// struct SampleItemC { +//! /// int value; +//! /// struct list_head link; +//! /// }; +//! /// ``` +//! #[repr(transparent)] +//! pub struct Item(Opaque); +//! +//! impl Item { +//! pub fn value(&self) -> i32 { +//! // SAFETY: `Item` has the same layout as `SampleItemC`. +//! unsafe { (*self.0.get()).value } +//! } +//! } +//! +//! // Create typed [`CList`] from sentinel head. +//! // SAFETY: `head` is valid and initialized, items are `SampleItemC` with +//! // embedded `link` field, and `Item` is `#[repr(transparent)]` over `SampleItemC`. +//! let list = unsafe { clist_create!(head, Item, SampleItemC, link) }; +//! +//! // Iterate directly over typed items. +//! let mut found_0 = false; +//! let mut found_10 = false; +//! let mut found_20 = false; +//! +//! for item in list.iter() { +//! let val = item.value(); +//! if val == 0 { found_0 = true; } +//! if val == 10 { found_10 = true; } +//! if val == 20 { found_20 = true; } +//! } +//! +//! assert!(found_0 && found_10 && found_20); +//! ``` + +use core::{ + iter::FusedIterator, + marker::PhantomData, // +}; + +use crate::{ + bindings, + types::Opaque, // +}; + +use pin_init::{ + pin_data, + pin_init, + PinInit, // +}; + +/// FFI wrapper for a C `list_head` object used in intrusive linked lists. +/// +/// # Invariants +/// +/// - The underlying `list_head` is initialized with valid non-`NULL` `next`/`prev` pointers. +#[pin_data] +#[repr(transparent)] +pub struct CListHead { + #[pin] + inner: Opaque, +} + +impl CListHead { + /// Create a `&CListHead` reference from a raw `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked `list_head` nodes must not be modified from + /// anywhere for the lifetime `'a`, unless done so via any [`CListHead`] APIs. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CListHead` has the same layout as `list_head`. + // - `ptr` is valid and unmodified for `'a` per caller guarantees. + unsafe { &*ptr.cast() } + } + + /// Get the raw `list_head` pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::list_head { + self.inner.get() + } + + /// Get the next [`CListHead`] in the list. + #[inline] + pub fn next(&self) -> &Self { + let raw = self.as_raw(); + // SAFETY: + // - `self.as_raw()` is valid and initialized per type invariants. + // - The `next` pointer is valid and non-`NULL` per type invariants + // (initialized via `INIT_LIST_HEAD()` or equivalent). + unsafe { Self::from_raw((*raw).next) } + } + + /// Check if this node is linked in a list (not isolated). + #[inline] + pub fn is_linked(&self) -> bool { + let raw = self.as_raw(); + // SAFETY: `self.as_raw()` is valid per type invariants. + unsafe { (*raw).next != raw && (*raw).prev != raw } + } + + /// Returns a pin-initializer for the list head. + pub fn new() -> impl PinInit { + pin_init!(Self { + // SAFETY: `INIT_LIST_HEAD` initializes `slot` to a valid empty list. + inner <- Opaque::ffi_init(|slot| unsafe { bindings::INIT_LIST_HEAD(slot) }), + }) + } +} + +// SAFETY: `list_head` contains no thread-bound state; it only holds +// `next`/`prev` pointers. +unsafe impl Send for CListHead {} + +// SAFETY: `CListHead` can be shared among threads as modifications are +// not allowed at the moment. +unsafe impl Sync for CListHead {} + +impl PartialEq for CListHead { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl Eq for CListHead {} + +/// Low-level iterator over `list_head` nodes. +/// +/// An iterator used to iterate over a C intrusive linked list (`list_head`). The caller has to +/// perform conversion of returned [`CListHead`] to an item (using [`container_of`] or similar). +/// +/// # Invariants +/// +/// `current` and `sentinel` are valid references into an initialized linked list. +struct CListHeadIter<'a> { + /// Current position in the list. + current: &'a CListHead, + /// The sentinel head (used to detect end of iteration). + sentinel: &'a CListHead, +} + +impl<'a> Iterator for CListHeadIter<'a> { + type Item = &'a CListHead; + + #[inline] + fn next(&mut self) -> Option { + // Check if we've reached the sentinel (end of list). + if self.current == self.sentinel { + return None; + } + + let item = self.current; + self.current = item.next(); + Some(item) + } +} + +impl<'a> FusedIterator for CListHeadIter<'a> {} + +/// A typed C linked list with a sentinel head intended for FFI use-cases where +/// a C subsystem manages a linked list that Rust code needs to read. Generally +/// required only for special cases. +/// +/// A sentinel head [`CListHead`] represents the entire linked list and can be used +/// for iteration over items of type `T`; it is not associated with a specific item. +/// +/// The const generic `OFFSET` specifies the byte offset of the `list_head` field within +/// the struct that `T` wraps. +/// +/// # Invariants +/// +/// - The sentinel [`CListHead`] has valid non-`NULL` `next`/`prev` pointers. +/// - `OFFSET` is the byte offset of the `list_head` field within the struct that `T` wraps. +/// - All the list's `list_head` nodes have valid non-`NULL` `next`/`prev` pointers. +#[repr(transparent)] +pub struct CList(CListHead, PhantomData); + +impl CList { + /// Create a typed [`CList`] reference from a raw sentinel `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized sentinel `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked nodes must not be concurrently modified for the lifetime `'a`. + /// - The list must contain items where the `list_head` field is at byte offset `OFFSET`. + /// - `T` must be `#[repr(transparent)]` over the C struct. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CList` has the same layout as `CListHead` due to `#[repr(transparent)]`. + // - Caller guarantees `ptr` is a valid, sentinel `list_head` object. + unsafe { &*ptr.cast() } + } + + /// Check if the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + !self.0.is_linked() + } + + /// Create an iterator over typed items. + #[inline] + pub fn iter(&self) -> CListIter<'_, T, OFFSET> { + let head = &self.0; + CListIter { + head_iter: CListHeadIter { + current: head.next(), + sentinel: head, + }, + _phantom: PhantomData, + } + } +} + +/// High-level iterator over typed list items. +pub struct CListIter<'a, T, const OFFSET: usize> { + head_iter: CListHeadIter<'a>, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T, const OFFSET: usize> Iterator for CListIter<'a, T, OFFSET> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option { + let head = self.head_iter.next()?; + + // Convert to item using `OFFSET`. + // + // SAFETY: The pointer calculation is valid because `OFFSET` is derived + // from `offset_of!` per type invariants. + Some(unsafe { &*head.as_raw().byte_sub(OFFSET).cast::() }) + } +} + +impl<'a, T, const OFFSET: usize> FusedIterator for CListIter<'a, T, OFFSET> {} + +/// Create a C doubly-circular linked list interface [`CList`] from a raw `list_head` pointer. +/// +/// This macro creates a `CList` that can iterate over items of type `$rust_type` +/// linked via the `$field` field in the underlying C struct `$c_type`. +/// +/// # Arguments +/// +/// - `$head`: Raw pointer to the sentinel `list_head` object (`*mut bindings::list_head`). +/// - `$rust_type`: Each item's Rust wrapper type. +/// - `$c_type`: Each item's C struct type that contains the embedded `list_head`. +/// - `$field`: The name of the `list_head` field within the C struct. +/// +/// # Safety +/// +/// The caller must ensure: +/// +/// - `$head` is a valid, initialized sentinel `list_head` (e.g. via `INIT_LIST_HEAD()`) +/// pointing to a list that is not concurrently modified for the lifetime of the [`CList`]. +/// - The list contains items of type `$c_type` linked via an embedded `$field`. +/// - `$rust_type` is `#[repr(transparent)]` over `$c_type` or has compatible layout. +/// +/// # Examples +/// +/// Refer to the examples in the [`crate::interop::list`] module documentation. +#[macro_export] +macro_rules! clist_create { + ($head:expr, $rust_type:ty, $c_type:ty, $($field:tt).+) => {{ + // Compile-time check that field path is a `list_head`. + let _: fn(*const $c_type) -> *const $crate::bindings::list_head = + |p| &raw const (*p).$($field).+; + + // Calculate offset and create `CList`. + const OFFSET: usize = ::core::mem::offset_of!($c_type, $($field).+); + $crate::interop::list::CList::<$rust_type, OFFSET>::from_raw($head) + }}; +} +pub use clist_create; diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000..a880ec3b8 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local processor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # ⚠️ Warning! ⚠️ +//! +//! The usage of this module can be more complicated than meets the eye, especially surrounding +//! [preemptible kernels]. It's recommended to take care when using the functions and types defined +//! here and familiarize yourself with the various documentation we have before using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preemptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible kernel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local +/// processor interrupt state management on their own and managing this by hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this + // object, hence calling the corresponding `local_interrupt_enable()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible kernels. + /// + /// This can be used for annotating code that is known to be run in contexts where local + /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs new file mode 100644 index 000000000..5ce9fd129 --- /dev/null +++ b/rust/kernel/io.rs @@ -0,0 +1,1754 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) + +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + +use crate::{ + bindings, + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // +}; + +#[cfg(CONFIG_HAS_IOMEM)] +pub mod mem; +pub mod poll; +pub mod register; +pub mod resource; + +pub use crate::register; +pub use resource::Resource; + +use register::LocatedRegister; + +/// Physical address type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type PhysAddr = bindings::phys_addr_t; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::resource_size_t; + +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region { + inner: [u8], +} + +impl Region { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl KnownSize for Region { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl IntoBytes for Region { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + +/// Raw representation of an MMIO region. +/// +/// `MmioRaw` is equivalent to `T __iomem *` in C. +/// +/// By itself, the existence of an instance of this structure does not provide any guarantees that +/// the represented MMIO region does exist or is properly mapped. +/// +/// Instead, the bus specific MMIO implementation must convert this raw representation into an +/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` +/// structure any guarantees are given. +pub struct MmioRaw { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, +} + +impl Copy for MmioRaw {} +impl Clone for MmioRaw { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Send for MmioRaw {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Sync for MmioRaw {} + +impl MmioRaw { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), + } + } +} + +impl MmioRaw> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) + } +} + +impl MmioRaw { + /// Returns the base address of the MMIO region. + #[inline] + pub fn addr(&self) -> usize { + self.ptr.addr() + } + + /// Returns the size of the MMIO region. + #[inline] + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) + } +} + +/// Checks whether an access of type `U` at the given `base` and the given `offset` +/// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. +#[inline] +const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::()) { + end <= size && (base.wrapping_add(offset) % align_of::() == 0) + } else { + false + } +} + +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> ::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + +/// Trait indicating that an I/O backend supports operations of a certain type and providing an +/// implementation for these operations. +/// +/// Different I/O backends can implement this trait to expose only the operations they support. +/// +/// For example, a PCI configuration space may implement `IoCapable`, `IoCapable`, +/// and `IoCapable`, but not `IoCapable`, while an MMIO region on a 64-bit +/// system might implement all four. +pub trait IoCapable: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; + + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); +} + +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); + + /// Copy contents from `buffer` to `view`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + let mut buf = MaybeUninit::::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } +} + +/// Describes a given I/O location: its offset, width, and type to convert the raw value from and +/// into. +/// +/// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and +/// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and +/// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets +/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] +/// macro). +/// +/// An `IoLoc` carries the following pieces of information: +/// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. +/// - The offset to access (returned by [`IoLoc::offset`]), +/// - The width of the access (determined by [`IoLoc::IoType`]), +/// - The type `T` in which the raw data is returned or provided. +/// +/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type +/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +pub trait IoLoc { + /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). + type IoType: Into + From; + + /// Consumes `self` and returns the offset of this location. + fn offset(self) -> usize; +} + +/// Implements [`IoLoc, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. +macro_rules! impl_usize_ioloc { + ($($ty:ty),*) => { + $( + impl IoLoc, $ty> for usize { + type IoType = $ty; + + #[inline(always)] + fn offset(self) -> usize { + self + } + } + )* + } +} + +// Provide the ability to read any primitive type from a [`usize`]. +impl_usize_ioloc!(u8, u16, u32, u64); + +/// Types implementing this trait (e.g. MMIO BARs or PCI config regions) +/// can perform I/O operations on regions of memory. +/// +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. +/// +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> ::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. +/// +/// This trait provides: +/// - Helper methods for offset validation and address calculation +/// - Fallible (runtime checked) accessors for different data widths +/// +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } + + /// Returns the length of the slice in number of elements. + #[inline] + fn len(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } + + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + fn try_cast(self) -> Result<::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::() > KnownSize::size(ptr) { + return Err(EINVAL); + } + + if ptr.addr() % align_of::() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } + } + + /// Fallible 8-bit read with runtime bounds check. + #[inline(always)] + fn try_read8(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 16-bit read with runtime bounds check. + #[inline(always)] + fn try_read16(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 32-bit read with runtime bounds check. + #[inline(always)] + fn try_read32(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 64-bit read with runtime bounds check. + #[inline(always)] + fn try_read64(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 8-bit write with runtime bounds check. + #[inline(always)] + fn try_write8(self, value: u8, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 16-bit write with runtime bounds check. + #[inline(always)] + fn try_write16(self, value: u16, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 32-bit write with runtime bounds check. + #[inline(always)] + fn try_write32(self, value: u32, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 64-bit write with runtime bounds check. + #[inline(always)] + fn try_write64(self, value: u64, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Infallible 8-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read8(self, offset: usize) -> u8 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 16-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read16(self, offset: usize) -> u16 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 32-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read32(self, offset: usize) -> u32 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 64-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read64(self, offset: usize) -> u64 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 8-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write8(self, value: u8, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 16-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write16(self, value: u16, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 32-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write32(self, value: u32, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 64-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write64(self, value: u64, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Generic fallible read with runtime bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region>) -> Result { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.try_read(0x10)?; + /// + /// // 8-bit read from address `0xfff`. + /// let v: u8 = io.try_read(0xfff)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_read(self, location: L) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view::(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) + } + + /// Generic fallible write with runtime bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region>) -> Result { + /// // 32-bit write of value `1` at address `0x10`. + /// io.try_write(0x10, 1u32)?; + /// + /// // 8-bit write of value `0xff` at address `0xfff`. + /// io.try_write(0xfff, 0xffu8)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_write(self, location: L, value: T) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view::(self, location.offset())?; + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + Ok(()) + } + + /// Generic fallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { + /// + /// io.try_write_reg(VERSION::new(1, 0)) + /// } + /// ``` + #[inline(always)] + fn try_write_reg(self, value: V) -> Result + where + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, + { + let (location, value) = value.into_io_op(); + + self.try_write(location, value) + } + + /// Generic fallible update with runtime bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { + /// io.try_update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn try_update(self, location: L, f: F) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + F: FnOnce(T) -> T, + { + let view = io_view::(self, location.offset())?; + + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); + + Ok(()) + } + + /// Generic infallible read with compile-time bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.read(0x10); + /// + /// // 8-bit read from the top of the I/O space. + /// let v: u8 = io.read(0xfff); + /// } + /// ``` + #[inline(always)] + fn read(self, location: L) -> T + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view_assert::(self, location.offset()); + Self::Backend::io_read(view).into() + } + + /// Generic infallible write with compile-time bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit write of value `1` at address `0x10`. + /// io.write(0x10, 1u32); + /// + /// // 8-bit write of value `0xff` at the top of the I/O space. + /// io.write(0xfff, 0xffu8); + /// } + /// ``` + #[inline(always)] + fn write(self, location: L, value: T) + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view_assert::(self, location.offset()); + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + } + + /// Generic infallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { + /// io.write_reg(VERSION::new(1, 0)); + /// } + /// ``` + #[inline(always)] + fn write_reg(self, value: V) + where + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, + { + let (location, value) = value.into_io_op(); + + self.write(location, value) + } + + /// Generic infallible update with compile-time bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { + /// io.update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn update(self, location: L, f: F) + where + L: IoLoc, + Self::Backend: IoCapable, + F: FnOnce(T) -> T, + { + let view = io_view_assert::(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); + } +} + +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + +/// A view of memory-mapped I/O region. +/// +/// # Invariant +/// +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for Mmio<'_, T> {} +impl Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for Mmio<'_, T> {} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_mmio_io_capable { + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } + } + + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } + } + } + }; +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); +// MMIO regions on 64-bit systems also support 64-bit accesses. +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); + +impl IoCopyable for MmioBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } + } +} + +/// [`Mmio`] but using relaxed accessors. +/// +/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of +/// the regular ones. +/// +/// See [`Mmio::relaxed`] for a usage example. +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); + +impl Copy for RelaxedMmio<'_, T> {} +impl Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. + /// + /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses + /// and can be used when such ordering is not required. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// RelaxedMmio, + /// }; + /// + /// fn do_io(io: Mmio<'_, Region<0x100>>) { + /// // The access is performed using `readl_relaxed` instead of `readl`. + /// let v = io.relaxed().read32(0x10); + /// } + /// + /// ``` + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) + } +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); +// MMIO regions on 64-bit systems also support 64-bit accesses. +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for SysMem<'_, T> {} +impl Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl Copy for IoSysMap<'_, T> {} +impl Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl IoCapable for IoSysMapBackend +where + MmioBackend: IoCapable, + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper(pub T); + +impl<'a, T> ProjectHelper +where + T: Io<'a, Backend: IoBackend = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view( + self, + ptr: *mut U, + ) -> ::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs new file mode 100644 index 000000000..32a919099 --- /dev/null +++ b/rust/kernel/io/mem.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic memory-mapped IO. + +use crate::{ + device::{ + Bound, + Device, // + }, + devres::DevresLt, + io::{ + self, + resource::{ + Region, + Resource, // + }, + IoBase, + Mmio, + MmioBackend, + MmioRaw, // + }, + prelude::*, + types::{ + CovariantForLt, + ForLt, // + }, +}; + +/// An IO request for a specific device and resource. +pub struct IoRequest<'a> { + device: &'a Device, + resource: &'a Resource, +} + +impl<'a> IoRequest<'a> { + /// Creates a new [`IoRequest`] instance. + /// + /// # Safety + /// + /// Callers must ensure that `resource` is valid for `device` during the + /// lifetime `'a`. + pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> Self { + IoRequest { device, resource } + } + + /// Maps an [`IoRequest`] where the size is known at compile time. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{ + /// bindings, + /// device::Core, + /// io::Io, + /// of, + /// platform, + /// }; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// # type Data<'bound> = Self; + /// + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { + /// let offset = 0; // Some offset. + /// + /// // If the size is known at compile time, use [`Self::iomap_sized`]. + /// // + /// // No runtime checks will apply when reading and writing. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap_sized::<42>()?; + /// + /// // Read and write a 32-bit value at `offset`. + /// let data = iomem.read32(offset); + /// + /// iomem.write32(data, offset); + /// + /// # Ok(SampleDriver) + /// } + /// } + /// ``` + pub fn iomap_sized(self) -> Result> { + IoMem::ioremap(self.device, self.resource) + } + + /// Same as [`Self::iomap_sized`] but with exclusive access to the + /// underlying region. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + pub fn iomap_exclusive_sized(self) -> Result> { + ExclusiveIoMem::ioremap(self.device, self.resource) + } + + /// Maps an [`IoRequest`] where the size is not known at compile time, + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{ + /// bindings, + /// device::Core, + /// io::Io, + /// of, + /// platform, + /// }; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// # type Data<'bound> = Self; + /// + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { + /// let offset = 0; // Some offset. + /// + /// // Unlike [`Self::iomap_sized`], here the size of the memory region + /// // is not known at compile time, so only the `try_read*` and `try_write*` + /// // family of functions should be used, leading to runtime checks on every + /// // access. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap()?; + /// + /// let data = iomem.try_read32(offset)?; + /// + /// iomem.try_write32(data, offset)?; + /// + /// # Ok(SampleDriver) + /// } + /// } + /// ``` + pub fn iomap(self) -> Result> { + self.iomap_sized::<0>() + } + + /// Same as [`Self::iomap`] but with exclusive access to the underlying + /// region. + pub fn iomap_exclusive(self) -> Result> { + self.iomap_exclusive_sized::<0>() + } +} + +/// An exclusive memory-mapped IO region. +/// +/// # Invariants +/// +/// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. +pub struct ExclusiveIoMem<'a, const SIZE: usize> { + /// The underlying `IoMem` instance. + iomem: IoMem<'a, SIZE>, + + /// The region abstraction. This represents exclusive access to the + /// range represented by the underlying `iomem`. + /// + /// This field is needed for ownership of the region. + _region: Region, +} + +impl ForLt for ExclusiveIoMem<'static, SIZE> { + type Of<'a> = ExclusiveIoMem<'a, SIZE>; +} + +// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`, +// which holds `&'a Device`, which is covariant. +unsafe impl CovariantForLt for ExclusiveIoMem<'static, SIZE> {} + +/// A device-managed exclusive I/O memory region. +/// +/// See [`ExclusiveIoMem::into_devres`]. +pub type DevresExclusiveIoMem = DevresLt>; + +impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { + /// Creates a new `ExclusiveIoMem` instance. + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { + let start = resource.start(); + let size = resource.size(); + let name = resource.name().unwrap_or_default(); + + let region = resource + .request_region( + start, + size, + name.to_cstring()?, + io::resource::Flags::IORESOURCE_MEM, + ) + .ok_or(EBUSY)?; + + let iomem = IoMem::ioremap(dev, resource)?; + + Ok(ExclusiveIoMem { + iomem, + _region: region, + }) + } + + /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in + /// driver data. Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.iomem.dev; + // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of + // which remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() + } +} + +/// A generic memory-mapped IO region. +/// +/// Accesses to the underlying region is checked either at compile time, if the +/// region's size is known at that point, or at runtime otherwise. +/// +/// # Invariants +/// +/// [`IoMem`] always holds an [`MmioRaw`] instance that holds a valid pointer to the +/// start of the I/O memory mapped region. +pub struct IoMem<'a, const SIZE: usize = 0> { + dev: &'a Device, + io: MmioRaw>, +} + +impl ForLt for IoMem<'static, SIZE> { + type Of<'a> = IoMem<'a, SIZE>; +} + +// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for IoMem<'static, SIZE> {} + +/// A device-managed I/O memory region. +/// +/// See [`IoMem::into_devres`]. +pub type DevresIoMem = DevresLt>; + +impl<'a, const SIZE: usize> IoMem<'a, SIZE> { + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { + // Note: Some ioremap() implementations use types that depend on the CPU + // word width rather than the bus address width. + // + // TODO: Properly address this in the C code to avoid this `try_into`. + let size = resource.size().try_into()?; + if size == 0 { + return Err(EINVAL); + } + + let res_start = resource.start(); + + let addr = if resource + .flags() + .contains(io::resource::Flags::IORESOURCE_MEM_NONPOSTED) + { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap_np(res_start, size) } + } else { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap(res_start, size) } + }; + + if addr.is_null() { + return Err(ENOMEM); + } + + let io = MmioRaw::new_region(addr as usize, size)?; + Ok(IoMem { dev, io }) + } + + /// Consume the `IoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data. + /// Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.dev; + // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } + } +} + +impl Drop for IoMem<'_, SIZE> { + fn drop(&mut self) { + // SAFETY: Safe as by the invariant of `Io`. + unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + // SAFETY: Safe as by the invariant of `IoMem`. + unsafe { Mmio::from_raw(self.io) } + } +} diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs new file mode 100644 index 000000000..d75f2fcf4 --- /dev/null +++ b/rust/kernel/io/poll.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IO polling. +//! +//! C header: [`include/linux/iopoll.h`](srctree/include/linux/iopoll.h). + +use crate::{ + prelude::*, + processor::cpu_relax, + task::might_sleep, + time::{ + delay::{ + fsleep, + udelay, // + }, + Delta, + Instant, + Monotonic, // + }, +}; + +/// Polls periodically until a condition is met, an error occurs, +/// or the timeout is reached. +/// +/// The function repeatedly executes the given operation `op` closure and +/// checks its result using the condition closure `cond`. +/// +/// If `cond` returns `true`, the function returns successfully with +/// the result of `op`. Otherwise, it waits for a duration specified +/// by `sleep_delta` before executing `op` again. +/// +/// This process continues until either `op` returns an error, `cond` +/// returns `true`, or the timeout specified by `timeout_delta` is +/// reached. +/// +/// This function can only be used in a nonatomic context. +/// +/// # Errors +/// +/// If `op` returns an error, then that error is returned directly. +/// +/// If the timeout specified by `timeout_delta` is reached, then +/// `Err(ETIMEDOUT)` is returned. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::io::{ +/// Io, +/// Mmio, +/// Region, +/// poll::read_poll_timeout, // +/// }; +/// use kernel::time::Delta; +/// +/// const HW_READY: u16 = 0x01; +/// +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { +/// read_poll_timeout( +/// // The `op` closure reads the value of a specific status register. +/// || io.try_read16(0x1000), +/// // The `cond` closure takes a reference to the value returned by `op` +/// // and checks whether the hardware is ready. +/// |val: &u16| *val == HW_READY, +/// Delta::from_millis(50), +/// Delta::from_secs(3), +/// )?; +/// Ok(()) +/// } +/// ``` +#[track_caller] +pub fn read_poll_timeout( + mut op: Op, + mut cond: Cond, + sleep_delta: Delta, + timeout_delta: Delta, +) -> Result +where + Op: FnMut() -> Result, + Cond: FnMut(&T) -> bool, +{ + let start: Instant = Instant::now(); + + // Unlike the C version, we always call `might_sleep()` unconditionally, + // as conditional calls are error-prone. We clearly separate + // `read_poll_timeout()` and `read_poll_timeout_atomic()` to aid + // tools like klint. + might_sleep(); + + loop { + let val = op()?; + if cond(&val) { + // Unlike the C version, we immediately return. + // We know the condition is met so we don't need to check again. + return Ok(val); + } + + if start.elapsed() > timeout_delta { + // Unlike the C version, we immediately return. + // We have just called `op()` so we don't need to call it again. + return Err(ETIMEDOUT); + } + + if !sleep_delta.is_zero() { + fsleep(sleep_delta); + } + + // `fsleep()` could be a busy-wait loop so we always call `cpu_relax()`. + cpu_relax(); + } +} + +/// Polls periodically until a condition is met, an error occurs, +/// or the attempt limit is reached. +/// +/// The function repeatedly executes the given operation `op` closure and +/// checks its result using the condition closure `cond`. +/// +/// If `cond` returns `true`, the function returns successfully with the result of `op`. +/// Otherwise, it performs a busy wait for a duration specified by `delay_delta` +/// before executing `op` again. +/// +/// This process continues until either `op` returns an error, `cond` +/// returns `true`, or the attempt limit specified by `retry` is reached. +/// +/// # Errors +/// +/// If `op` returns an error, then that error is returned directly. +/// +/// If the attempt limit specified by `retry` is reached, then +/// `Err(ETIMEDOUT)` is returned. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::io::{ +/// Io, +/// Mmio, +/// Region, +/// poll::read_poll_timeout_atomic, // +/// }; +/// use kernel::time::Delta; +/// +/// const HW_READY: u16 = 0x01; +/// +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { +/// read_poll_timeout_atomic( +/// // The `op` closure reads the value of a specific status register. +/// || io.try_read16(0x1000), +/// // The `cond` closure takes a reference to the value returned by `op` +/// // and checks whether the hardware is ready. +/// |val: &u16| *val == HW_READY, +/// Delta::from_micros(50), +/// 1000, +/// )?; +/// Ok(()) +/// } +/// ``` +pub fn read_poll_timeout_atomic( + mut op: Op, + mut cond: Cond, + delay_delta: Delta, + retry: usize, +) -> Result +where + Op: FnMut() -> Result, + Cond: FnMut(&T) -> bool, +{ + for _ in 0..retry { + let val = op()?; + if cond(&val) { + return Ok(val); + } + + if !delay_delta.is_zero() { + udelay(delay_delta); + } + + cpu_relax(); + } + + Err(ETIMEDOUT) +} diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs new file mode 100644 index 000000000..03dfd2ff4 --- /dev/null +++ b/rust/kernel/io/register.rs @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Macro to define register layout and accessors. +//! +//! The [`register!`](kernel::io::register!) macro provides an intuitive and readable syntax for +//! defining a dedicated type for each register and accessing it using [`Io`](super::Io). Each such +//! type comes with its own field accessors that can return an error if a field's value is invalid. +//! +//! Note: most of the items in this module are public so they can be referenced by the macro, but +//! most are not to be used directly by users. Outside of the `register!` macro itself, the only +//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! +//! # Simple example +//! +//! ```no_run +//! use kernel::io::register; +//! +//! register! { +//! /// Basic information about the chip. +//! pub BOOT_0(u32) @ 0x00000100 { +//! /// Vendor ID. +//! 15:8 vendor_id; +//! /// Major revision of the chip. +//! 7:4 major_revision; +//! /// Minor revision of the chip. +//! 3:0 minor_revision; +//! } +//! } +//! ``` +//! +//! This defines a 32-bit `BOOT_0` type which can be read from or written to offset `0x100` of an +//! `Io` region, with the described bitfields. For instance, `minor_revision` consists of the 4 +//! least significant bits of the type. +//! +//! Fields are instances of [`Bounded`](kernel::num::Bounded) and can be read by calling their +//! getter method, which is named after them. They also have setter methods prefixed with `with_` +//! for runtime values and `with_const_` for constant values. All setters return the updated +//! register value. +//! +//! Fields can also be transparently converted from/to an arbitrary type by using the `=>` and +//! `?=>` syntaxes. +//! +//! If present, doc comments above register or fields definitions are added to the relevant item +//! they document (the register type itself, or the field's setter and getter methods). +//! +//! Note that multiple registers can be defined in a single `register!` invocation. This can be +//! useful to group related registers together. +//! +//! Here is how the register defined above can be used in code: +//! +//! +//! ```no_run +//! use kernel::{ +//! io::{ +//! register, +//! Io, +//! IoLoc, +//! }, +//! num::Bounded, +//! }; +//! # use kernel::io::{Mmio, Region}; +//! # register! { +//! # pub BOOT_0(u32) @ 0x00000100 { +//! # 15:8 vendor_id; +//! # 7:4 major_revision; +//! # 3:0 minor_revision; +//! # } +//! # } +//! # fn test(io: Mmio<'_, Region<0x1000>>) { +//! # fn obtain_vendor_id() -> u8 { 0xff } +//! +//! // Read from the register's defined offset (0x100). +//! let boot0 = io.read(BOOT_0); +//! pr_info!("chip revision: {}.{}", boot0.major_revision().get(), boot0.minor_revision().get()); +//! +//! // Update some fields and write the new value back. +//! let new_boot0 = boot0 +//! // Constant values. +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! // Runtime value. +//! .with_vendor_id(obtain_vendor_id()); +//! io.write_reg(new_boot0); +//! +//! // Or, build a new value from zero and write it: +//! io.write_reg(BOOT_0::zeroed() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Or, read and update the register in a single step. +//! io.update(BOOT_0, |r| r +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Constant values can also be built using the const setters. +//! const V: BOOT_0 = pin_init::zeroed::() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>(); +//! # } +//! ``` +//! +//! For more extensive documentation about how to define registers, see the +//! [`register!`](kernel::io::register!) macro. + +use core::marker::PhantomData; + +use crate::{ + build_assert::build_assert, + io::IoLoc, // +}; + +use super::Region; + +/// Trait implemented by all registers. +pub trait Register: Sized { + /// Backing primitive type of the register. + type Storage: Into + From; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; +} + +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Register {} + +/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when +/// passing a [`FixedRegister`] value. +impl IoLoc, T> for () +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used +/// as an [`IoLoc`]. +impl IoLoc, T> for T +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Location of a fixed register. +pub struct FixedRegisterLoc(PhantomData); + +impl FixedRegisterLoc { + /// Returns the location of `T`. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl IoLoc, T> for FixedRegisterLoc +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Trait providing a base address to be added to the offset of a relative register to obtain +/// its actual offset. +/// +/// The `T` generic argument is used to distinguish which base to use, in case a type provides +/// several bases. It is given to the `register!` macro to restrict the use of the register to +/// implementors of this particular variant. +pub trait RegisterBase { + /// Base address to which register offsets are added. + const BASE: usize; +} + +/// Trait implemented by all registers that are relative to a base. +pub trait WithBase { + /// Family of bases applicable to this register. + type BaseFamily; + + /// Returns the absolute location of this type when using `B` as its base. + #[inline(always)] + fn of>() -> RelativeRegisterLoc + where + Self: Register, + { + RelativeRegisterLoc::new() + } +} + +/// Trait implemented by relative registers. +pub trait RelativeRegister: Register + WithBase {} + +/// Location of a relative register. +/// +/// This can either be an immediately accessible regular [`RelativeRegister`], or a +/// [`RelativeRegisterArray`] that needs one additional resolution through +/// [`RelativeRegisterLoc::at`]. +pub struct RelativeRegisterLoc(PhantomData, PhantomData); + +impl RelativeRegisterLoc +where + T: Register + WithBase, + B: RegisterBase + ?Sized, +{ + /// Returns the location of a relative register or register array. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData, PhantomData) + } + + // Returns the absolute offset of the relative register using base `B`. + // + // This is implemented as a private const method so it can be reused by the [`IoLoc`] + // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. + #[inline] + const fn offset(self) -> usize { + B::BASE + T::OFFSET + } +} + +impl IoLoc, T> for RelativeRegisterLoc +where + T: RelativeRegister, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + RelativeRegisterLoc::offset(self) + } +} + +/// Trait implemented by arrays of registers. +pub trait RegisterArray: Register { + /// Number of elements in the registers array. + const SIZE: usize; + /// Number of bytes between the start of elements in the registers array. + const STRIDE: usize; +} + +/// Location of an array register. +pub struct RegisterArrayLoc(usize, PhantomData); + +impl RegisterArrayLoc { + /// Returns the location of register `T` at position `idx`, with build-time validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(idx, PhantomData) + } + + /// Attempts to return the location of register `T` at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(idx, PhantomData)) + } else { + None + } + } +} + +impl IoLoc, T> for RegisterArrayLoc +where + T: RegisterArray, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + self.0 * T::STRIDE + } +} + +/// Trait providing location builders for [`RegisterArray`]s. +pub trait Array { + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + fn at(idx: usize) -> RegisterArrayLoc + where + Self: RegisterArray, + { + RegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + fn try_at(idx: usize) -> Option> + where + Self: RegisterArray, + { + RegisterArrayLoc::try_new(idx) + } +} + +/// Trait implemented by arrays of relative registers. +pub trait RelativeRegisterArray: RegisterArray + WithBase {} + +/// Location of a relative array register. +pub struct RelativeRegisterArrayLoc< + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +>(RelativeRegisterLoc, usize); + +impl RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of register `T` from the base `B` at index `idx`, with build-time + /// validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(RelativeRegisterLoc::new(), idx) + } + + /// Attempts to return the location of register `T` from the base `B` at index `idx`, with + /// runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(RelativeRegisterLoc::new(), idx)) + } else { + None + } + } +} + +/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. +impl RelativeRegisterLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc { + RelativeRegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_at(self, idx: usize) -> Option> { + RelativeRegisterArrayLoc::try_new(idx) + } +} + +impl IoLoc, T> for RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + self.0.offset() + self.1 * T::STRIDE + } +} + +/// Trait implemented by items that contain both a register value and the absolute I/O location at +/// which to write it. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait LocatedRegister { + /// Register value to write. + type Value: Register; + /// Full location information at which to write the value. + type Location: IoLoc; + + /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write + /// operation. + fn into_io_op(self) -> (Self::Location, Self::Value); +} + +impl LocatedRegister> for T +where + T: FixedRegister, +{ + type Location = FixedRegisterLoc; + type Value = T; + + #[inline(always)] + fn into_io_op(self) -> (FixedRegisterLoc, T) { + (FixedRegisterLoc::new(), self) + } +} + +/// Defines a dedicated type for a register, including getter and setter methods for its fields and +/// methods to read and write it from an [`Io`](kernel::io::Io) region. +/// +/// This documentation focuses on how to declare registers. See the [module-level +/// documentation](mod@kernel::io::register) for examples of how to access them. +/// +/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of +/// registers, and relative arrays of registers. +/// +/// ## Fixed offset registers +/// +/// These are the simplest kind of registers. Their location is simply an offset inside the I/O +/// region. For instance: +/// +/// ```ignore +/// register! { +/// pub FIXED_REG(u16) @ 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a 16-bit register named `FIXED_REG` located at offset `0x80` of an I/O region. +/// +/// These registers' location can be built simply by referencing their name: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// register! { +/// FIXED_REG(u32) @ 0x100 { +/// 15:8 high_byte; +/// 7:0 low_byte; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// let val = io.read(FIXED_REG); +/// +/// // Write from an already-existing value. +/// io.write(FIXED_REG, val.with_low_byte(0xff)); +/// +/// // Create a register value from scratch. +/// let val2 = FIXED_REG::zeroed().with_high_byte(0x80); +/// +/// // The location of fixed offset registers is already contained in their type. Thus, the +/// // `location` argument of `Io::write` is technically redundant and can be replaced by `()`. +/// io.write((), val2); +/// +/// // Or, the single-argument `Io::write_reg` can be used. +/// io.write_reg(val2); +/// # } +/// +/// ``` +/// +/// It is possible to create an alias of an existing register with new field definitions by using +/// the `=> ALIAS` syntax. This is useful for cases where a register's interpretation depends on +/// the context: +/// +/// ```no_run +/// use kernel::io::register; +/// +/// register! { +/// /// Scratch register. +/// pub SCRATCH(u32) @ 0x00000200 { +/// 31:0 value; +/// } +/// +/// /// Boot status of the firmware. +/// pub SCRATCH_BOOT_STATUS(u32) => SCRATCH { +/// 0:0 completed; +/// } +/// } +/// ``` +/// +/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing +/// its own `completed` field. +/// +/// ## Relative registers +/// +/// Relative registers can be instantiated several times at a relative offset of a group of bases. +/// For instance, imagine the following I/O space: +/// +/// ```text +/// +-----------------------------+ +/// | ... | +/// | | +/// 0x100--->+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ +/// ``` +/// +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done using the `Base + Offset` syntax when specifying the register's address: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REG(u32) @ Base + 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a register with an offset of `0x80` from a given base. +/// +/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the +/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for +/// this register needs to implement `RegisterBase`. +/// +/// The location of relative registers can be built using the [`WithBase::of`] method to specify +/// its base. All relative registers implement [`WithBase`]. +/// +/// Here is the above layout translated into code: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// // Type used to identify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. +/// register! { +/// /// CPU core control. +/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { +/// 0:0 start; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io.read(CPU_CTL::of::()); +/// +/// // Stop `Cpu0`. +/// io.write(WithBase::of::(), CPU_CTL::zeroed()); +/// # } +/// +/// // Aliases can also be defined for relative register. +/// register! { +/// /// Alias to CPU core control. +/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { +/// /// Start the aliased CPU core. +/// 1:1 alias_start; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { +/// // Start the aliased `CPU0`, leaving its other fields untouched. +/// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); +/// # } +/// ``` +/// +/// ## Arrays of registers +/// +/// Some I/O areas contain consecutive registers that share the same field layout. These areas can +/// be defined as an array of identical registers, allowing them to be accessed by index with +/// compile-time or runtime bound checking: +/// +/// ```ignore +/// register! { +/// pub REGISTER_ARRAY(u8)[10, stride = 4] @ 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// This defines `REGISTER_ARRAY`, an array of 10 byte registers starting at offset `0x100`. Each +/// register is separated from its neighbor by 4 bytes. +/// +/// The `stride` parameter is optional; if unspecified, the registers are placed consecutively from +/// each other. +/// +/// A location for a register in a register array is built using the [`Array::at`] trait method. +/// All arrays of registers implement [`Array`]. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::Array, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. +/// register! { +/// /// Scratch registers. +/// pub SCRATCH(u32)[64] @ 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) +/// # -> Result<(), Error>{ +/// // Read scratch register 0, i.e. I/O address `0x80`. +/// let scratch_0 = io.read(SCRATCH::at(0)).value(); +/// +/// // Write scratch register 15, i.e. I/O address `0x80 + (15 * 4)`. +/// io.write(Array::at(15), SCRATCH::from(0xffeeaabb)); +/// +/// // This is out of bounds and won't build. +/// // let scratch_128 = io.read(SCRATCH::at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let some_scratch = io.read(SCRATCH::try_at(idx).ok_or(EINVAL)?).value(); +/// +/// // Alias to a specific register in an array. +/// // Here `SCRATCH[8]` is used to convey the firmware exit code. +/// register! { +/// /// Firmware exit status code. +/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// let status = io.read(FIRMWARE_STATUS).status(); +/// +/// // Non-contiguous register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ 0x000000c4 { +/// 31:0 value; +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Relative arrays of registers +/// +/// Combining the two features described in the sections above, arrays of registers accessible from +/// a base can also be defined: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// Like relative registers, they implement the [`WithBase`] trait. However the return value of +/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the +/// [`at`](RelativeRegisterLoc::at) method. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Type used as parameter of `RegisterBase` to specify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // 64 per-cpu scratch registers, arranged as a contiguous array. +/// register! { +/// /// Per-CPU scratch registers. +/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// // Read scratch register 0 of CPU0. +/// let scratch = io.read(CPU_SCRATCH::of::().at(0)); +/// +/// // Write the retrieved value into scratch register 15 of CPU1. +/// io.write(WithBase::of::().at(15), scratch); +/// +/// // This won't build. +/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::().at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let scratch_idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let cpu0_scratch = io.read( +/// CPU_SCRATCH::of::().try_at(scratch_idx).ok_or(EINVAL)? +/// ).value(); +/// # Ok(()) +/// # } +/// +/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// register! { +/// /// Per-CPU firmware exit status code. +/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); +/// # Ok(()) +/// # } +/// ``` +#[macro_export] +macro_rules! register { + // Entry point for the macro, allowing multiple registers to be defined in one call. + // It matches all possible register declaration patterns to dispatch them to corresponding + // `@reg` rule that defines a single register. + // + // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O + // projections. + ( + $( + $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + $([ $size:expr $(, stride = $stride:expr)? ])? + $(@ $($base:ident +)? $offset:literal)? + $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? + { $($fields:tt)* } + )* + ) => { + $( + $crate::register!( + @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? + $(@ $($base +)? $offset)? + $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? + { $($fields)* } + ); + )* + }; + + // All the rules below are private helpers. + + // Creates a register at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an alias register of fixed offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates a register at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an alias register of relative offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an array of registers at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_array $name [ $size, stride = $stride ]); + }; + + // Shortcut for contiguous array of registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of array of registers `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] + { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an array of registers at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); + }; + + // Shortcut for contiguous array of relative registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $base + $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of relative array of registers `alias` with its own + // fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Generates the bitfield for the register. + // + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + ( + @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!( + #[allow(non_camel_case_types)] + $(#[$attr])* $vis struct $name($storage) { $($fields)* } + ); + }; + + // Implementations shared by all registers types. + (@io_base $name:ident($storage:ty) @ $offset:expr) => { + impl $crate::io::register::Register for $name { + type Storage = $storage; + + const OFFSET: usize = $offset; + } + }; + + // Implementations of fixed registers. + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { + impl $crate::io::register::FixedRegister for $name {} + + $(#[$attr])* + $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = + $crate::io::register::FixedRegisterLoc::<$name>::new(); + }; + + // Implementations of relative registers. + (@io_relative $name:ident @ $base:ident) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RelativeRegister for $name {} + }; + + // Implementations of register arrays. + (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { + impl $crate::io::register::Array for $name {} + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + }; + + // Implementations of relative array registers. + ( + @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident + ) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + + impl $crate::io::register::RelativeRegisterArray for $name {} + }; +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs new file mode 100644 index 000000000..17b0c174c --- /dev/null +++ b/rust/kernel/io/resource.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for [system +//! resources](https://docs.kernel.org/core-api/kernel-api.html#resources-management). +//! +//! C header: [`include/linux/ioport.h`](srctree/include/linux/ioport.h) + +use core::{ + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + str::CString, + types::Opaque, // +}; + +pub use super::{ + PhysAddr, + ResourceSize, // +}; + +/// A region allocated from a parent [`Resource`]. +/// +/// # Invariants +/// +/// - `self.0` points to a valid `bindings::resource` that was obtained through +/// `bindings::__request_region`. +pub struct Region { + /// The resource returned when the region was requested. + resource: NonNull, + /// The name that was passed in when the region was requested. We need to + /// store it for ownership reasons. + _name: CString, +} + +impl Deref for Region { + type Target = Resource; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as per the invariant of `Region`. + unsafe { Resource::from_raw(self.resource.as_ptr()) } + } +} + +impl Drop for Region { + fn drop(&mut self) { + let (flags, start, size) = { + let res = &**self; + (res.flags(), res.start(), res.size()) + }; + + let release_fn = if flags.contains(Flags::IORESOURCE_MEM) { + bindings::release_mem_region + } else { + bindings::release_region + }; + + // SAFETY: Safe as per the invariant of `Region`. + unsafe { release_fn(start, size) }; + } +} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, which is safe to be used from +// any thread. +unsafe impl Send for Region {} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, references to which are +// safe to be used from any thread. +unsafe impl Sync for Region {} + +/// A resource abstraction. +/// +/// # Invariants +/// +/// [`Resource`] is a transparent wrapper around a valid `bindings::resource`. +#[repr(transparent)] +pub struct Resource(Opaque); + +impl Resource { + /// Creates a reference to a [`Resource`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of 'a, the pointer will + /// point at a valid `bindings::resource`. + /// + /// The caller must also ensure that the [`Resource`] is only accessed via the + /// returned reference for the duration of 'a. + pub(crate) const unsafe fn from_raw<'a>(ptr: *mut bindings::resource) -> &'a Self { + // SAFETY: Self is a transparent wrapper around `Opaque`. + unsafe { &*ptr.cast() } + } + + /// Requests a resource region. + /// + /// Exclusive access will be given and the region will be marked as busy. + /// Further calls to [`Self::request_region`] will return [`None`] if + /// the region, or a part of it, is already in use. + pub fn request_region( + &self, + start: PhysAddr, + size: ResourceSize, + name: CString, + flags: Flags, + ) -> Option { + // SAFETY: + // - Safe as per the invariant of `Resource`. + // - `__request_region` will store a reference to the name, but that is + // safe as we own it and it will not be dropped until the `Region` is + // dropped. + let region = unsafe { + bindings::__request_region( + self.0.get(), + start, + size, + name.as_char_ptr(), + flags.0 as c_int, + ) + }; + + Some(Region { + resource: NonNull::new(region)?, + _name: name, + }) + } + + /// Returns the size of the resource. + pub fn size(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { bindings::resource_size(inner) } + } + + /// Returns the start address of the resource. + pub fn start(&self) -> PhysAddr { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { (*inner).start } + } + + /// Returns the name of the resource. + pub fn name(&self) -> Option<&CStr> { + let inner = self.0.get(); + + // SAFETY: Safe as per the invariants of `Resource`. + let name = unsafe { (*inner).name }; + + if name.is_null() { + return None; + } + + // SAFETY: In the C code, `resource::name` either contains a null + // pointer or points to a valid NUL-terminated C string, and at this + // point we know it is not null, so we can safely convert it to a + // `CStr`. + Some(unsafe { CStr::from_char_ptr(name) }) + } + + /// Returns the flags associated with the resource. + pub fn flags(&self) -> Flags { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + let flags = unsafe { (*inner).flags }; + + Flags(flags) + } +} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, which is +// safe to be used from any thread. +unsafe impl Send for Resource {} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, references +// to which are safe to be used from any thread. +unsafe impl Sync for Resource {} + +/// Resource flags as stored in the C `struct resource::flags` field. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the associated constants such as +/// [`Flags::IORESOURCE_IO`]. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +impl Flags { + /// PCI/ISA I/O ports. + pub const IORESOURCE_IO: Flags = Flags::new(bindings::IORESOURCE_IO); + + /// Resource is software muxed. + pub const IORESOURCE_MUXED: Flags = Flags::new(bindings::IORESOURCE_MUXED); + + /// Resource represents a memory region. + pub const IORESOURCE_MEM: Flags = Flags::new(bindings::IORESOURCE_MEM); + + /// Resource represents a memory region that must be ioremaped using `ioremap_np`. + pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn new(value: u32) -> Self { + build_assert!(value as u64 <= c_ulong::MAX as u64); + Flags(value as c_ulong) + } +} diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs new file mode 100644 index 000000000..5bb5b48cf --- /dev/null +++ b/rust/kernel/ioctl.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! `ioctl()` number definitions. +//! +//! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h) + +#![expect(non_snake_case)] + +use crate::build_assert::build_assert; + +/// Build an ioctl number, analogous to the C macro of the same name. +#[inline(always)] +const fn _IOC(dir: u32, ty: u32, nr: u32, size: usize) -> u32 { + build_assert!(dir <= uapi::_IOC_DIRMASK); + build_assert!(ty <= uapi::_IOC_TYPEMASK); + build_assert!(nr <= uapi::_IOC_NRMASK); + build_assert!(size <= (uapi::_IOC_SIZEMASK as usize)); + + (dir << uapi::_IOC_DIRSHIFT) + | (ty << uapi::_IOC_TYPESHIFT) + | (nr << uapi::_IOC_NRSHIFT) + | ((size as u32) << uapi::_IOC_SIZESHIFT) +} + +/// Build an ioctl number for an argumentless ioctl. +#[inline(always)] +pub const fn _IO(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_NONE, ty, nr, 0) +} + +/// Build an ioctl number for a read-only ioctl. +#[inline(always)] +pub const fn _IOR(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for a write-only ioctl. +#[inline(always)] +pub const fn _IOW(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for a read-write ioctl. +#[inline(always)] +pub const fn _IOWR(ty: u32, nr: u32) -> u32 { + _IOC( + uapi::_IOC_READ | uapi::_IOC_WRITE, + ty, + nr, + core::mem::size_of::(), + ) +} + +/// Get the ioctl direction from an ioctl number. +pub const fn _IOC_DIR(nr: u32) -> u32 { + (nr >> uapi::_IOC_DIRSHIFT) & uapi::_IOC_DIRMASK +} + +/// Get the ioctl type from an ioctl number. +pub const fn _IOC_TYPE(nr: u32) -> u32 { + (nr >> uapi::_IOC_TYPESHIFT) & uapi::_IOC_TYPEMASK +} + +/// Get the ioctl number from an ioctl number. +pub const fn _IOC_NR(nr: u32) -> u32 { + (nr >> uapi::_IOC_NRSHIFT) & uapi::_IOC_NRMASK +} + +/// Get the ioctl size from an ioctl number. +pub const fn _IOC_SIZE(nr: u32) -> usize { + ((nr >> uapi::_IOC_SIZESHIFT) & uapi::_IOC_SIZEMASK) as usize +} diff --git a/rust/kernel/iommu/mod.rs b/rust/kernel/iommu/mod.rs new file mode 100644 index 000000000..1423d7b19 --- /dev/null +++ b/rust/kernel/iommu/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust support related to IOMMU. + +pub mod pgtable; diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs new file mode 100644 index 000000000..5f9b42ca9 --- /dev/null +++ b/rust/kernel/iommu/pgtable.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IOMMU page table management. +//! +//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h) + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; + +use crate::{ + alloc, + bindings, + device::{ + Bound, + Device, // + }, + error::to_result, + io::PhysAddr, + prelude::*, // +}; + +use bindings::io_pgtable_fmt; + +/// Protection flags used with IOMMU mappings. +pub mod prot { + /// Read access. + pub const READ: u32 = bindings::IOMMU_READ; + /// Write access. + pub const WRITE: u32 = bindings::IOMMU_WRITE; + /// Request cache coherency. + pub const CACHE: u32 = bindings::IOMMU_CACHE; + /// Request no-execute permission. + pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; + /// MMIO peripheral mapping. + pub const MMIO: u32 = bindings::IOMMU_MMIO; + /// Privileged mapping. + pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV; +} + +/// Represents a requested `io_pgtable` configuration. +pub struct Config { + /// Quirk bitmask (type-specific). + pub quirks: usize, + /// Valid page sizes, as a bitmask of powers of two. + pub pgsize_bitmap: usize, + /// Input address space size in bits. + pub ias: u32, + /// Output address space size in bits. + pub oas: u32, + /// IOMMU uses coherent accesses for page table walks. + pub coherent_walk: bool, +} + +/// An io page table using a specific format. +/// +/// # Invariants +/// +/// The pointer references a valid io page table. +pub struct IoPageTable<'a, F: IoPageTableFmt> { + ptr: NonNull, + _dev: PhantomData<&'a Device>, + _marker: PhantomData, +} + +// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. +unsafe impl Send for IoPageTable<'_, F> {} +// SAFETY: `struct io_pgtable_ops` may be accessed concurrently. +unsafe impl Sync for IoPageTable<'_, F> {} + +/// The format used by this page table. +pub trait IoPageTableFmt: 'static { + /// The value representing this format. + const FORMAT: io_pgtable_fmt; +} + +impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { + /// Create a new `IoPageTable`. + #[inline] + pub fn new(dev: &'a Device, config: Config) -> Result> { + let mut raw_cfg = bindings::io_pgtable_cfg { + quirks: config.quirks, + pgsize_bitmap: config.pgsize_bitmap, + ias: config.ias, + oas: config.oas, + coherent_walk: config.coherent_walk, + tlb: &raw const NOOP_FLUSH_OPS, + iommu_dev: dev.as_raw(), + ..Zeroable::zeroed() + }; + + // SAFETY: + // * The raw_cfg pointer is valid for the duration of this call. + // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer + // as cookie. + // * The caller ensures that the io pgtable does not outlive the device. + let ops = unsafe { + bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut()) + }; + + // INVARIANT: We successfully created a valid page table. + Ok(IoPageTable { + ptr: NonNull::new(ops).ok_or(ENOMEM)?, + _dev: PhantomData, + _marker: PhantomData, + }) + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`. + #[inline] + pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops { + self.ptr.as_ptr() + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable`. + #[inline] + pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable { + // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable. + unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) } + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`. + #[inline] + pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg { + // SAFETY: The `raw_pgtable()` method returns a valid pointer. + unsafe { &raw mut (*self.raw_pgtable()).cfg } + } + + /// Map a physically contiguous range of pages of the same size. + /// + /// Even if successful, this operation may not map the entire range. In that case, only a + /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In + /// this case, the caller will usually call `map_pages` again for the remaining range. + /// + /// The returned [`Result`] indicates whether an error was encountered while mapping pages. + /// Note that this may return a non-zero length even if an error was encountered. The caller + /// will usually [unmap the relevant pages](Self::unmap_pages) on error. + /// + /// The caller must flush the TLB before using the pgtable to access the newly created mapping. + /// + /// # Safety + /// + /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while + /// this `map_pages` operation executes. + /// * This page table must not contain any mapping that overlaps with the mapping created by + /// this call. + /// * If this page table is live, then the caller must ensure that it's okay to access the + /// physical address being mapped for the duration in which it is mapped. + #[inline] + pub unsafe fn map_pages( + &self, + iova: usize, + paddr: PhysAddr, + pgsize: usize, + pgcount: usize, + prot: u32, + flags: alloc::Flags, + ) -> (usize, Result) { + let mut mapped: usize = 0; + + // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null. + let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() }; + + // SAFETY: The safety requirements of this method are sufficient to call `map_pages`. + let ret = to_result(unsafe { + (map_pages)( + self.raw_ops(), + iova, + paddr, + pgsize, + pgcount, + prot as i32, + flags.as_raw(), + &mut mapped, + ) + }); + + (mapped, ret) + } + + /// Unmap a range of virtually contiguous pages of the same size. + /// + /// This may not unmap the entire range, and returns the length of the unmapped prefix in + /// bytes. + /// + /// # Safety + /// + /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while + /// this `unmap_pages` operation executes. + /// * This page table must contain one or more consecutive mappings starting at `iova` whose + /// total size is `pgcount * pgsize`. + #[inline] + #[must_use] + pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize { + // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null. + let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() }; + + // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`. + unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) } + } +} + +// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all +// required invalidations using a range. There is no need for it get ARM style invalidation +// instructions from the page table code. +// +// Support for flushing the TLB with ARM style invalidation instructions may be added in the +// future. +static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { + tlb_flush_all: Some(rust_tlb_flush_all_noop), + tlb_flush_walk: Some(rust_tlb_flush_walk_noop), + tlb_add_page: None, +}; + +#[no_mangle] +extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {} + +#[no_mangle] +extern "C" fn rust_tlb_flush_walk_noop( + _iova: usize, + _size: usize, + _granule: usize, + _cookie: *mut core::ffi::c_void, +) { +} + +impl Drop for IoPageTable<'_, F> { + fn drop(&mut self) { + // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this + // destructor runs. + unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) }; + } +} + +/// The `ARM_64_LPAE_S1` page table format. +pub enum ARM64LPAES1 {} + +impl IoPageTableFmt for ARM64LPAES1 { + const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; +} + +impl IoPageTable<'_, ARM64LPAES1> { + /// Access the `ttbr` field of the configuration. + /// + /// This is the physical address of the page table, which may be passed to the device that + /// needs to use it. + /// + /// # Safety + /// + /// The caller must ensure that the device stops using the page table before dropping it. + #[inline] + pub unsafe fn ttbr(&self) -> u64 { + // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. + unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr } + } + + /// Access the `mair` field of the configuration. + #[inline] + pub fn mair(&self) -> u64 { + // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. + unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair } + } +} diff --git a/rust/kernel/iov.rs b/rust/kernel/iov.rs new file mode 100644 index 000000000..43bae8923 --- /dev/null +++ b/rust/kernel/iov.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! IO vectors. +//! +//! C headers: [`include/linux/iov_iter.h`](srctree/include/linux/iov_iter.h), +//! [`include/linux/uio.h`](srctree/include/linux/uio.h) + +use crate::{ + alloc::{Allocator, Flags}, + bindings, + prelude::*, + types::Opaque, +}; +use core::{marker::PhantomData, mem::MaybeUninit, ptr, slice}; + +const ITER_SOURCE: bool = bindings::ITER_SOURCE != 0; +const ITER_DEST: bool = bindings::ITER_DEST != 0; + +// Compile-time assertion for the above constants. +const _: () = { + build_assert!( + ITER_SOURCE != ITER_DEST, + "ITER_DEST and ITER_SOURCE should be different." + ); +}; + +/// An IO vector that acts as a source of data. +/// +/// The data may come from many different sources. This includes both things in kernel-space and +/// reading from userspace. It's not necessarily the case that the data source is immutable, so +/// rewinding the IO vector to read the same data twice is not guaranteed to result in the same +/// bytes. It's also possible that the data source is mapped in a thread-local manner using e.g. +/// `kmap_local_page()`, so this type is not `Send` to ensure that the mapping is read from the +/// right context in that scenario. +/// +/// # Invariants +/// +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_SOURCE`. For the duration +/// of `'data`, it must be safe to read from this IO vector using the standard C methods for this +/// purpose. +#[repr(transparent)] +pub struct IovIterSource<'data> { + iov: Opaque, + /// Represent to the type system that this value contains a pointer to readable data it does + /// not own. + _source: PhantomData<&'data [u8]>, +} + +impl<'data> IovIterSource<'data> { + /// Obtain an `IovIterSource` from a raw pointer. + /// + /// # Safety + /// + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the + /// returned reference for the duration of `'iov`. + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_SOURCE`. + /// * For the duration of `'data`, it must be safe to read from this IO vector using the + /// standard C methods for this purpose. + #[track_caller] + #[inline] + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterSource<'data> { + // SAFETY: The caller ensures that `ptr` is valid. + let data_source = unsafe { (*ptr).data_source }; + assert_eq!(data_source, ITER_SOURCE); + + // SAFETY: The caller ensures the type invariants for the right durations, and + // `IovIterSource` is layout compatible with `struct iov_iter`. + unsafe { &mut *ptr.cast::>() } + } + + /// Access this as a raw `struct iov_iter`. + #[inline] + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { + self.iov.get() + } + + /// Returns the number of bytes available in this IO vector. + /// + /// Note that this may overestimate the number of bytes. For example, reading from userspace + /// memory could fail with `EFAULT`, which will be treated as the end of the IO vector. + #[inline] + pub fn len(&self) -> usize { + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. + unsafe { + (*self.iov.get()) + .__bindgen_anon_1 + .__bindgen_anon_1 + .as_ref() + .count + } + } + + /// Returns whether there are any bytes left in this IO vector. + /// + /// This may return `true` even if there are no more bytes available. For example, reading from + /// userspace memory could fail with `EFAULT`, which will be treated as the end of the IO vector. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Advance this IO vector by `bytes` bytes. + /// + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. + #[inline] + pub fn advance(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; + } + + /// Advance this IO vector backwards by `bytes` bytes. + /// + /// # Safety + /// + /// The IO vector must not be reverted to before its beginning. + #[inline] + pub unsafe fn revert(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller + // ensures that `bytes` is in bounds. + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; + } + + /// Read data from this IO vector. + /// + /// Returns the number of bytes that have been copied. + #[inline] + pub fn copy_from_iter(&mut self, out: &mut [u8]) -> usize { + // SAFETY: `Self::copy_from_iter_raw` guarantees that it will not write any uninitialized + // bytes in the provided buffer, so `out` is still a valid `u8` slice after this call. + let out = unsafe { &mut *(ptr::from_mut(out) as *mut [MaybeUninit]) }; + + self.copy_from_iter_raw(out).len() + } + + /// Read data from this IO vector and append it to a vector. + /// + /// Returns the number of bytes that have been copied. + #[inline] + pub fn copy_from_iter_vec( + &mut self, + out: &mut Vec, + flags: Flags, + ) -> Result { + out.reserve(self.len(), flags)?; + let len = self.copy_from_iter_raw(out.spare_capacity_mut()).len(); + // SAFETY: + // - `len` is the length of a subslice of the spare capacity, so `len` is at most the + // length of the spare capacity. + // - `Self::copy_from_iter_raw` guarantees that the first `len` bytes of the spare capacity + // have been initialized. + unsafe { out.inc_len(len) }; + Ok(len) + } + + /// Read data from this IO vector into potentially uninitialized memory. + /// + /// Returns the sub-slice of the output that has been initialized. If the returned slice is + /// shorter than the input buffer, then the entire IO vector has been read. + /// + /// This will never write uninitialized bytes to the provided buffer. + #[inline] + pub fn copy_from_iter_raw(&mut self, out: &mut [MaybeUninit]) -> &mut [u8] { + let capacity = out.len(); + let out = out.as_mut_ptr().cast::(); + + // GUARANTEES: The C API guarantees that it does not write uninitialized bytes to the + // provided buffer. + // SAFETY: + // * By the type invariants, it is still valid to read from this IO vector. + // * `out` is valid for writing for `capacity` bytes because it comes from a slice of + // that length. + let len = unsafe { bindings::_copy_from_iter(out.cast(), capacity, self.as_raw()) }; + + // SAFETY: The underlying C api guarantees that initialized bytes have been written to the + // first `len` bytes of the spare capacity. + unsafe { slice::from_raw_parts_mut(out, len) } + } +} + +/// An IO vector that acts as a destination for data. +/// +/// IO vectors support many different types of destinations. This includes both buffers in +/// kernel-space and writing to userspace. It's possible that the destination buffer is mapped in a +/// thread-local manner using e.g. `kmap_local_page()`, so this type is not `Send` to ensure that +/// the mapping is written to the right context in that scenario. +/// +/// # Invariants +/// +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_DEST`. For the duration of +/// `'data`, it must be safe to write to this IO vector using the standard C methods for this +/// purpose. +#[repr(transparent)] +pub struct IovIterDest<'data> { + iov: Opaque, + /// Represent to the type system that this value contains a pointer to writable data it does + /// not own. + _source: PhantomData<&'data mut [u8]>, +} + +impl<'data> IovIterDest<'data> { + /// Obtain an `IovIterDest` from a raw pointer. + /// + /// # Safety + /// + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the + /// returned reference for the duration of `'iov`. + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_DEST`. + /// * For the duration of `'data`, it must be safe to write to this IO vector using the + /// standard C methods for this purpose. + #[track_caller] + #[inline] + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterDest<'data> { + // SAFETY: The caller ensures that `ptr` is valid. + let data_source = unsafe { (*ptr).data_source }; + assert_eq!(data_source, ITER_DEST); + + // SAFETY: The caller ensures the type invariants for the right durations, and + // `IovIterSource` is layout compatible with `struct iov_iter`. + unsafe { &mut *ptr.cast::>() } + } + + /// Access this as a raw `struct iov_iter`. + #[inline] + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { + self.iov.get() + } + + /// Returns the number of bytes available in this IO vector. + /// + /// Note that this may overestimate the number of bytes. For example, reading from userspace + /// memory could fail with EFAULT, which will be treated as the end of the IO vector. + #[inline] + pub fn len(&self) -> usize { + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. + unsafe { + (*self.iov.get()) + .__bindgen_anon_1 + .__bindgen_anon_1 + .as_ref() + .count + } + } + + /// Returns whether there are any bytes left in this IO vector. + /// + /// This may return `true` even if there are no more bytes available. For example, reading from + /// userspace memory could fail with EFAULT, which will be treated as the end of the IO vector. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Advance this IO vector by `bytes` bytes. + /// + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. + #[inline] + pub fn advance(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; + } + + /// Advance this IO vector backwards by `bytes` bytes. + /// + /// # Safety + /// + /// The IO vector must not be reverted to before its beginning. + #[inline] + pub unsafe fn revert(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller + // ensures that `bytes` is in bounds. + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; + } + + /// Write data to this IO vector. + /// + /// Returns the number of bytes that were written. If this is shorter than the provided slice, + /// then no more bytes can be written. + #[inline] + pub fn copy_to_iter(&mut self, input: &[u8]) -> usize { + // SAFETY: + // * By the type invariants, it is still valid to write to this IO vector. + // * `input` is valid for `input.len()` bytes. + unsafe { bindings::_copy_to_iter(input.as_ptr().cast(), input.len(), self.as_raw()) } + } + + /// Utility for implementing `read_iter` given the full contents of the file. + /// + /// The full contents of the file being read from is represented by `contents`. This call will + /// write the appropriate sub-slice of `contents` and update the file position in `ppos` so + /// that the file will appear to contain `contents` even if takes multiple reads to read the + /// entire file. + #[inline] + pub fn simple_read_from_buffer(&mut self, ppos: &mut i64, contents: &[u8]) -> Result { + if *ppos < 0 { + return Err(EINVAL); + } + let Ok(pos) = usize::try_from(*ppos) else { + return Ok(0); + }; + if pos >= contents.len() { + return Ok(0); + } + + // BOUNDS: We just checked that `pos < contents.len()` above. + let num_written = self.copy_to_iter(&contents[pos..]); + + // OVERFLOW: `pos+num_written <= contents.len() <= isize::MAX <= i64::MAX`. + *ppos = (pos + num_written) as i64; + + Ok(num_written) + } +} diff --git a/rust/kernel/irq.rs b/rust/kernel/irq.rs new file mode 100644 index 000000000..09ef1e7f8 --- /dev/null +++ b/rust/kernel/irq.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IRQ abstractions. +//! +//! An IRQ is an interrupt request from a device. It is used to get the CPU's +//! attention so it can service a hardware event in a timely manner. +//! +//! The current abstractions handle IRQ requests and handlers, i.e.: it allows +//! drivers to register a handler for a given IRQ line. +//! +//! C header: [`include/linux/interrupt.h`](srctree/include/linux/interrupt.h) + +/// Flags to be used when registering IRQ handlers. +mod flags; + +/// IRQ allocation and handling. +mod request; + +pub use flags::Flags; + +pub use request::{ + Handler, IrqRequest, IrqReturn, Registration, ThreadedHandler, ThreadedIrqReturn, + ThreadedRegistration, +}; diff --git a/rust/kernel/irq/flags.rs b/rust/kernel/irq/flags.rs new file mode 100644 index 000000000..d26e25af0 --- /dev/null +++ b/rust/kernel/irq/flags.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright 2025 Collabora ltd. + +use crate::bindings; +use crate::prelude::*; + +/// Flags to be used when registering IRQ handlers. +/// +/// Flags can be used to request specific behaviors when registering an IRQ +/// handler, and can be combined using the `|`, `&`, and `!` operators to +/// further control the system's behavior. +/// +/// A common use case is to register a shared interrupt, as sharing the line +/// between devices is increasingly common in modern systems and is even +/// required for some buses. This requires setting [`Flags::SHARED`] when +/// requesting the interrupt. Other use cases include setting the trigger type +/// through `Flags::TRIGGER_*`, which determines when the interrupt fires, or +/// controlling whether the interrupt is masked after the handler runs by using +/// [`Flags::ONESHOT`]. +/// +/// If an invalid combination of flags is provided, the system will refuse to +/// register the handler, and lower layers will enforce certain flags when +/// necessary. This means, for example, that all the +/// [`crate::irq::Registration`] for a shared interrupt have to agree on +/// [`Flags::SHARED`] and on the same trigger type, if set. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Use the interrupt line as already configured. + pub const TRIGGER_NONE: Flags = Flags::new(bindings::IRQF_TRIGGER_NONE); + + /// The interrupt is triggered when the signal goes from low to high. + pub const TRIGGER_RISING: Flags = Flags::new(bindings::IRQF_TRIGGER_RISING); + + /// The interrupt is triggered when the signal goes from high to low. + pub const TRIGGER_FALLING: Flags = Flags::new(bindings::IRQF_TRIGGER_FALLING); + + /// The interrupt is triggered while the signal is held high. + pub const TRIGGER_HIGH: Flags = Flags::new(bindings::IRQF_TRIGGER_HIGH); + + /// The interrupt is triggered while the signal is held low. + pub const TRIGGER_LOW: Flags = Flags::new(bindings::IRQF_TRIGGER_LOW); + + /// Allow sharing the IRQ among several devices. + pub const SHARED: Flags = Flags::new(bindings::IRQF_SHARED); + + /// Set by callers when they expect sharing mismatches to occur. + pub const PROBE_SHARED: Flags = Flags::new(bindings::IRQF_PROBE_SHARED); + + /// Flag to mark this interrupt as timer interrupt. + pub const TIMER: Flags = Flags::new(bindings::IRQF_TIMER); + + /// Interrupt is per CPU. + pub const PERCPU: Flags = Flags::new(bindings::IRQF_PERCPU); + + /// Flag to exclude this interrupt from irq balancing. + pub const NOBALANCING: Flags = Flags::new(bindings::IRQF_NOBALANCING); + + /// Interrupt is used for polling (only the interrupt that is registered + /// first in a shared interrupt is considered for performance reasons). + pub const IRQPOLL: Flags = Flags::new(bindings::IRQF_IRQPOLL); + + /// Interrupt is not re-enabled after the hardirq handler finished. Used by + /// threaded interrupts which need to keep the irq line disabled until the + /// threaded handler has been run. + pub const ONESHOT: Flags = Flags::new(bindings::IRQF_ONESHOT); + + /// Do not disable this IRQ during suspend. Does not guarantee that this + /// interrupt will wake the system from a suspended state. + pub const NO_SUSPEND: Flags = Flags::new(bindings::IRQF_NO_SUSPEND); + + /// Force enable it on resume even if [`Flags::NO_SUSPEND`] is set. + pub const FORCE_RESUME: Flags = Flags::new(bindings::IRQF_FORCE_RESUME); + + /// Interrupt cannot be threaded. + pub const NO_THREAD: Flags = Flags::new(bindings::IRQF_NO_THREAD); + + /// Resume IRQ early during syscore instead of at device resume time. + pub const EARLY_RESUME: Flags = Flags::new(bindings::IRQF_EARLY_RESUME); + + /// If the IRQ is shared with a [`Flags::NO_SUSPEND`] user, execute this + /// interrupt handler after suspending interrupts. For system wakeup devices + /// users need to implement wakeup detection in their interrupt handlers. + pub const COND_SUSPEND: Flags = Flags::new(bindings::IRQF_COND_SUSPEND); + + /// Don't enable IRQ or NMI automatically when users request it. Users will + /// enable it explicitly by `enable_irq` or `enable_nmi` later. + pub const NO_AUTOEN: Flags = Flags::new(bindings::IRQF_NO_AUTOEN); + + /// Exclude from runnaway detection for IPI and similar handlers, depends on + /// `PERCPU`. + pub const NO_DEBUG: Flags = Flags::new(bindings::IRQF_NO_DEBUG); + + pub(crate) fn into_inner(self) -> c_ulong { + self.0 + } + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn new(value: u32) -> Self { + build_assert!(value as u64 <= c_ulong::MAX as u64); + Self(value as c_ulong) + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs new file mode 100644 index 000000000..c1c6525a6 --- /dev/null +++ b/rust/kernel/irq/request.rs @@ -0,0 +1,483 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright 2025 Collabora ltd. + +//! This module provides types like [`Registration`] and +//! [`ThreadedRegistration`], which allow users to register handlers for a given +//! IRQ line. + +use core::marker::{ + PhantomData, + PhantomPinned, // +}; + +use crate::{ + device::{ + Bound, + Device, // + }, + error::to_result, + irq::flags::Flags, + prelude::*, + str::CStr, +}; + +/// The value that can be returned from a [`Handler`] or a [`ThreadedHandler`]. +#[repr(u32)] +pub enum IrqReturn { + /// The interrupt was not from this device or was not handled. + None = bindings::irqreturn_IRQ_NONE, + + /// The interrupt was handled by this device. + Handled = bindings::irqreturn_IRQ_HANDLED, +} + +/// Callbacks for an IRQ handler. +pub trait Handler: Sync { + /// The hard IRQ handler. + /// + /// This is executed in interrupt context, hence all corresponding + /// limitations do apply. + /// + /// All work that does not necessarily need to be executed from + /// interrupt context, should be deferred to a threaded handler. + /// See also [`ThreadedRegistration`]. + fn handle(&self) -> IrqReturn; +} + +/// A request for an IRQ line for a given device. +/// +/// # Invariants +/// +/// - `ìrq` is the number of an interrupt source of `dev`. +/// - `irq` has not been registered yet; this is consumed by [`Registration::new()`]. +pub struct IrqRequest<'a> { + irq: u32, + /// Proves the device is bound at registration time and ties `'a` to the device's bound + /// lifetime, ensuring the [`Registration`] cannot outlive it. + _dev: PhantomData<&'a Device>, +} + +impl<'a> IrqRequest<'a> { + /// Creates a new IRQ request for the given device and IRQ number. + /// + /// # Safety + /// + /// - `irq` should be a valid IRQ number for `dev`. + pub(crate) unsafe fn new(_dev: &'a Device, irq: u32) -> Self { + // INVARIANT: `irq` is a valid IRQ number for `dev`. + IrqRequest { + irq, + _dev: PhantomData, + } + } + + /// Returns the IRQ number of an [`IrqRequest`]. + #[inline] + pub fn irq(&self) -> u32 { + self.irq + } +} + +/// A registration of an IRQ handler for a given IRQ line. +/// +/// # Examples +/// +/// The following is an example of using `Registration`. It uses a +/// [`Completion`] to coordinate between the IRQ +/// handler and process context. [`Completion`] uses interior mutability, so the +/// handler can signal with [`Completion::complete_all()`] and the process +/// context can wait with [`Completion::wait_for_completion()`] even though +/// there is no way to get a mutable reference to the any of the fields in +/// `Data`. +/// +/// [`Completion`]: kernel::sync::Completion +/// [`Completion::complete_all()`]: kernel::sync::Completion::complete_all +/// [`Completion::wait_for_completion()`]: kernel::sync::Completion::wait_for_completion +/// +/// ``` +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// Registration, +/// }, +/// prelude::*, +/// sync::Completion, +/// }; +/// +/// // Data shared between process and IRQ context. +/// #[pin_data] +/// struct Data { +/// #[pin] +/// completion: Completion, +/// } +/// +/// impl irq::Handler for Data { +/// // Executed in IRQ context. +/// fn handle(&self) -> IrqReturn { +/// self.completion.complete_all(); +/// IrqReturn::Handled +/// } +/// } +/// +/// // Registers an IRQ handler for the given IrqRequest. +/// // +/// // This runs in process context and assumes `request` was previously acquired from a device. +/// fn register_irq( +/// request: IrqRequest<'_>, +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// Registration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// completion <- Completion::new(), +/// }? Error), +/// ) +/// }; +/// +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; +/// +/// registration.handler().completion.wait_for_completion(); +/// +/// Ok(registration) +/// } +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * We own an irq handler registered via `request_irq` whose cookie is a pointer to `Self`. +#[pin_data(PinnedDrop)] +pub struct Registration<'a, T: Handler> { + request: IrqRequest<'a>, + + #[pin] + handler: T, + + /// Pinned because we need address stability so that we can pass a pointer + /// to the callback. + #[pin] + _pin: PhantomPinned, +} + +impl<'a, T: Handler> Registration<'a, T> { + /// Registers the IRQ handler with the system for the given IRQ number. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn new( + request: IrqRequest<'a>, + flags: Flags, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_irq` whose cookie is a pointer to `Self`. + try_pin_init!(&this in Self { + handler <- handler, + request, + _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callback to be + // called immediately. + to_result(unsafe { + bindings::request_irq( + request.irq, + Some(handle_irq_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, + }) + } + + /// Returns a reference to the handler that was registered with the system. + pub fn handler(&self) -> &T { + &self.handler + } + + /// Wait for pending IRQ handlers on other CPUs. + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `Registration::new()`. This blocks + // until all in-flight handlers complete, so no references to `self` remain after this + // returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; + } +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_irq`. +unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. + let registration = unsafe { &*ptr }; + + T::handle(®istration.handler) as c_uint +} + +/// The value that can be returned from [`ThreadedHandler::handle`]. +#[repr(u32)] +pub enum ThreadedIrqReturn { + /// The interrupt was not from this device or was not handled. + None = bindings::irqreturn_IRQ_NONE, + + /// The interrupt was handled by this device. + Handled = bindings::irqreturn_IRQ_HANDLED, + + /// The handler wants the handler thread to wake up. + WakeThread = bindings::irqreturn_IRQ_WAKE_THREAD, +} + +/// Callbacks for a threaded IRQ handler. +pub trait ThreadedHandler: Sync { + /// The hard IRQ handler. + /// + /// This is executed in interrupt context, hence all corresponding + /// limitations do apply. All work that does not necessarily need to be + /// executed from interrupt context, should be deferred to the threaded + /// handler, i.e. [`ThreadedHandler::handle_threaded`]. + /// + /// The default implementation returns [`ThreadedIrqReturn::WakeThread`]. + fn handle(&self) -> ThreadedIrqReturn { + ThreadedIrqReturn::WakeThread + } + + /// The threaded IRQ handler. + /// + /// This is executed in process context. The kernel creates a dedicated + /// `kthread` for this purpose. + fn handle_threaded(&self) -> IrqReturn; +} + +/// A registration of a threaded IRQ handler for a given IRQ line. +/// +/// Two callbacks are required: one to handle the IRQ, and one to handle any +/// other work in a separate thread. +/// +/// The thread handler is only called if the IRQ handler returns +/// [`ThreadedIrqReturn::WakeThread`]. +/// +/// # Examples +/// +/// The following is an example of using [`ThreadedRegistration`]. It uses a +/// [`Mutex`](kernel::sync::Mutex) to provide interior mutability. +/// +/// ``` +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// ThreadedHandler, +/// ThreadedIrqReturn, +/// ThreadedRegistration, +/// }, +/// prelude::*, +/// sync::Mutex, +/// }; +/// +/// // Declare a struct that will be passed in when the interrupt fires. The u32 +/// // merely serves as an example of some internal data. +/// // +/// // [`irq::ThreadedHandler::handle`] takes `&self`. This example +/// // illustrates how interior mutability can be used when sharing the data +/// // between process context and IRQ context. +/// #[pin_data] +/// struct Data { +/// #[pin] +/// value: Mutex, +/// } +/// +/// impl ThreadedHandler for Data { +/// // This will run (in a separate kthread) if and only if +/// // [`ThreadedHandler::handle`] returns [`WakeThread`], which it does by +/// // default. +/// fn handle_threaded(&self) -> IrqReturn { +/// let mut data = self.value.lock(); +/// *data += 1; +/// IrqReturn::Handled +/// } +/// } +/// +/// // Registers a threaded IRQ handler for the given [`IrqRequest`]. +/// // +/// // This is executing in process context and assumes that `request` was +/// // previously acquired from a device. +/// fn register_threaded_irq( +/// request: IrqRequest<'_>, +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// ThreadedRegistration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// value <- kernel::new_mutex!(0), +/// }? Error), +/// ) +/// }; +/// +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; +/// +/// { +/// // The data can be accessed from process context too. +/// let mut data = registration.handler().value.lock(); +/// *data += 1; +/// } +/// +/// Ok(registration) +/// } +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * We own an irq handler registered via `request_threaded_irq` whose cookie is a pointer to +/// `Self`. +#[pin_data(PinnedDrop)] +pub struct ThreadedRegistration<'a, T: ThreadedHandler> { + request: IrqRequest<'a>, + + #[pin] + handler: T, + + /// Pinned because we need address stability so that we can pass a pointer + /// to the callback. + #[pin] + _pin: PhantomPinned, +} + +impl<'a, T: ThreadedHandler> ThreadedRegistration<'a, T> { + /// Registers the IRQ handler with the system for the given IRQ number. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`ThreadedRegistration`] or otherwise prevent + /// its [`Drop`] implementation from running. + pub unsafe fn new( + request: IrqRequest<'a>, + flags: Flags, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_threaded_irq` whose cookie is a pointer to `Self`. + try_pin_init!(&this in Self { + handler <- handler, + request, + _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_threaded_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callbacks to be + // called immediately. + to_result(unsafe { + bindings::request_threaded_irq( + request.irq, + Some(handle_threaded_irq_callback::), + Some(thread_fn_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, + }) + } + + /// Returns a reference to the handler that was registered with the system. + pub fn handler(&self) -> &T { + &self.handler + } + + /// Wait for pending IRQ handlers on other CPUs. + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; + } +} + +#[pinned_drop] +impl PinnedDrop for ThreadedRegistration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `ThreadedRegistration::new()`. This + // blocks until all in-flight handlers complete, so no references to `self` remain after + // this returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; + } +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_threaded_irq`. +unsafe extern "C" fn handle_threaded_irq_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; + + T::handle(®istration.handler) as c_uint +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_threaded_irq`. +unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; + + T::handle_threaded(®istration.handler) as c_uint +} diff --git a/rust/kernel/jump_label.rs b/rust/kernel/jump_label.rs new file mode 100644 index 000000000..f54cedcb6 --- /dev/null +++ b/rust/kernel/jump_label.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for static keys. +//! +//! C header: [`include/linux/jump_label.h`](srctree/include/linux/jump_label.h). + +/// Branch based on a static key. +/// +/// Takes three arguments: +/// +/// * `key` - the path to the static variable containing the `static_key`. +/// * `keytyp` - the type of `key`. +/// * `field` - the name of the field of `key` that contains the `static_key`. +/// +/// # Safety +/// +/// The macro must be used with a real static key defined by C. +#[macro_export] +macro_rules! static_branch_unlikely { + ($key:path, $keytyp:ty, $field:ident) => {{ + let _key: *const $keytyp = ::core::ptr::addr_of!($key); + let _key: *const $crate::bindings::static_key_false = ::core::ptr::addr_of!((*_key).$field); + let _key: *const $crate::bindings::static_key = _key.cast(); + + #[cfg(not(CONFIG_JUMP_LABEL))] + { + $crate::bindings::static_key_count(_key.cast_mut()) > 0 + } + + #[cfg(CONFIG_JUMP_LABEL)] + $crate::jump_label::arch_static_branch! { $key, $keytyp, $field, false } + }}; +} +pub use static_branch_unlikely; + +/// Assert that the assembly block evaluates to a string literal. +#[cfg(CONFIG_JUMP_LABEL)] +const _: &str = include!(concat!( + env!("OBJTREE"), + "/rust/kernel/generated_arch_static_branch_asm.rs" +)); + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: { + $crate::asm!( + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_static_branch_asm.rs")); + l_yes = label { + break 'my_label true; + }, + symb = sym $key, + off = const ::core::mem::offset_of!($keytyp, $field), + branch = const $crate::jump_label::bool_to_int($branch), + ); + + break 'my_label false; + }}; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(testlib)] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => { + // The asm falls through until patched, which never happens on the host. + false + }; +} + +#[cfg(CONFIG_JUMP_LABEL)] +pub use arch_static_branch; + +/// A helper used by inline assembly to pass a boolean to as a `const` parameter. +/// +/// Using this function instead of a cast lets you assert that the input is a boolean, and not some +/// other type that can also be cast to an integer. +#[doc(hidden)] +pub const fn bool_to_int(b: bool) -> i32 { + b as i32 +} diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs new file mode 100644 index 000000000..91eaff8c1 --- /dev/null +++ b/rust/kernel/kunit.rs @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! KUnit-based macros for Rust unit tests. +//! +//! C header: [`include/kunit/test.h`](srctree/include/kunit/test.h) +//! +//! Reference: + +use crate::fmt; +use crate::prelude::*; + +/// Prints a KUnit error-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn err(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + c"\x013%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Prints a KUnit info-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn info(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + c"\x016%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Asserts that a boolean expression is `true` at runtime. +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert { + ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + 'out: { + // Do nothing if the condition is `true`. + if $condition { + break 'out; + } + + static FILE: &'static $crate::str::CStr = $file; + static LINE: i32 = ::core::line!() as i32 - $diff; + static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); + + // SAFETY: FFI call without safety requirements. + let kunit_test = unsafe { $crate::bindings::kunit_get_current_test() }; + if kunit_test.is_null() { + // The assertion failed but this task is not running a KUnit test, so we cannot call + // KUnit, but at least print an error to the kernel log. This may happen if this + // macro is called from an spawned thread in a test (see + // `scripts/rustdoc_test_gen.rs`) or if some non-test code calls this macro by + // mistake (it is hidden to prevent that). + // + // This mimics KUnit's failed assertion format. + $crate::kunit::err($crate::prelude::fmt!( + " # {}: ASSERTION FAILED at {FILE}:{LINE}\n", + $name + )); + $crate::kunit::err($crate::prelude::fmt!( + " Expected {CONDITION} to be true, but is false\n" + )); + $crate::kunit::err($crate::prelude::fmt!( + " Failure not reported to KUnit since this is a non-KUnit task\n" + )); + break 'out; + } + + #[repr(transparent)] + struct Location($crate::bindings::kunit_loc); + + #[repr(transparent)] + struct UnaryAssert($crate::bindings::kunit_unary_assert); + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for Location {} + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for UnaryAssert {} + + static LOCATION: Location = Location($crate::bindings::kunit_loc { + file: $crate::str::as_char_ptr_in_const_context(FILE), + line: LINE, + }); + static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert { + assert: $crate::bindings::kunit_assert {}, + condition: $crate::str::as_char_ptr_in_const_context(CONDITION), + expected_true: true, + }); + + // SAFETY: + // - FFI call. + // - The `kunit_test` pointer is valid because we got it from + // `kunit_get_current_test()` and it was not null. This means we are in a KUnit + // test, and that the pointer can be passed to KUnit functions and assertions. + // - The string pointers (`file` and `condition` above) point to null-terminated + // strings since they are `CStr`s. + // - The function pointer (`format`) points to the proper function. + // - The pointers passed will remain valid since they point to `static`s. + // - The format string is allowed to be null. + // - There are, however, problems with this: first of all, this will end up stopping + // the thread, without running destructors. While that is problematic in itself, + // it is considered UB to have what is effectively a forced foreign unwind + // with `extern "C"` ABI. One could observe the stack that is now gone from + // another thread. We should avoid pinning stack variables to prevent library UB, + // too. For the moment, given that test failures are reported immediately before the + // next test runs, that test failures should be fixed and that KUnit is explicitly + // documented as not suitable for production environments, we feel it is reasonable. + unsafe { + $crate::bindings::__kunit_do_failed_assertion( + kunit_test, + ::core::ptr::addr_of!(LOCATION.0), + $crate::bindings::kunit_assert_type_KUNIT_ASSERTION, + ::core::ptr::addr_of!(ASSERTION.0.assert), + Some($crate::bindings::kunit_unary_assert_format), + ::core::ptr::null(), + ); + } + + // SAFETY: FFI call; the `test` pointer is valid because this hidden macro should only + // be called by the generated documentation tests which forward the test pointer given + // by KUnit. + unsafe { + $crate::bindings::__kunit_abort(kunit_test); + } + } + }; +} + +/// Asserts that two expressions are equal to each other (using [`PartialEq`]). +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert_eq { + ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + // For the moment, we just forward to the expression assert because, for binary asserts, + // KUnit supports only a few types (e.g. integers). + $crate::kunit_assert!($name, $file, $diff, $left == $right); + }}; +} + +trait TestResult { + fn is_test_result_ok(&self) -> bool; +} + +impl TestResult for () { + fn is_test_result_ok(&self) -> bool { + true + } +} + +impl TestResult for Result { + fn is_test_result_ok(&self) -> bool { + self.is_ok() + } +} + +/// Returns whether a test result is to be considered OK. +/// +/// This will be `assert!`ed from the generated tests. +#[doc(hidden)] +#[expect(private_bounds)] +pub fn is_test_result_ok(t: impl TestResult) -> bool { + t.is_test_result_ok() +} + +/// Represents an individual test case. +#[doc(hidden)] +pub const fn kunit_case( + name: &'static kernel::str::CStr, + run_case: unsafe extern "C" fn(*mut kernel::bindings::kunit), +) -> kernel::bindings::kunit_case { + kernel::bindings::kunit_case { + run_case: Some(run_case), + name: kernel::str::as_char_ptr_in_const_context(name), + attr: kernel::bindings::kunit_attributes { + speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + generate_params: None, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + module_name: core::ptr::null_mut(), + log: core::ptr::null_mut(), + param_init: None, + param_exit: None, + } +} + +/// Registers a KUnit test suite. +/// +/// # Safety +/// +/// `test_cases` must be a `NULL` terminated array of valid test cases, +/// whose lifetime is at least that of the test suite (i.e., static). +/// +/// # Examples +/// +/// ```ignore +/// extern "C" fn test_fn(_test: *mut kernel::bindings::kunit) { +/// let actual = 1 + 1; +/// let expected = 2; +/// assert_eq!(actual, expected); +/// } +/// +/// static mut KUNIT_TEST_CASES: [kernel::bindings::kunit_case; 2] = [ +/// kernel::kunit::kunit_case(c"name", test_fn), +/// pin_init::zeroed(), +/// ]; +/// kernel::kunit_unsafe_test_suite!(suite_name, KUNIT_TEST_CASES); +/// ``` +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_unsafe_test_suite { + ($name:ident, $test_cases:ident) => { + const _: () = { + const KUNIT_TEST_SUITE_NAME: [::kernel::ffi::c_char; 256] = { + let name_u8 = ::core::stringify!($name).as_bytes(); + let mut ret = [0; 256]; + + if name_u8.len() > 255 { + panic!(concat!( + "The test suite name `", + ::core::stringify!($name), + "` exceeds the maximum length of 255 bytes." + )); + } + + let mut i = 0; + while i < name_u8.len() { + ret[i] = name_u8[i] as ::kernel::ffi::c_char; + i += 1; + } + + ret + }; + + static mut KUNIT_TEST_SUITE: ::kernel::bindings::kunit_suite = + ::kernel::bindings::kunit_suite { + name: KUNIT_TEST_SUITE_NAME, + #[allow(unused_unsafe)] + // SAFETY: `$test_cases` is passed in by the user, and + // (as documented) must be valid for the lifetime of + // the suite (i.e., static). + test_cases: unsafe { + ::core::ptr::addr_of_mut!($test_cases) + .cast::<::kernel::bindings::kunit_case>() + }, + suite_init: None, + suite_exit: None, + init: None, + exit: None, + attr: ::kernel::bindings::kunit_attributes { + speed: ::kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + status_comment: [0; 256usize], + debugfs: ::core::ptr::null_mut(), + log: ::core::ptr::null_mut(), + suite_init_err: 0, + is_init: false, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + }; + + #[used(compiler)] + #[allow(unused_unsafe)] + #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")] + static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite = + // SAFETY: `KUNIT_TEST_SUITE` is static. + unsafe { ::core::ptr::addr_of_mut!(KUNIT_TEST_SUITE) }; + }; + }; +} + +/// Returns whether we are currently running a KUnit test. +/// +/// In some cases, you need to call test-only code from outside the test case, for example, to +/// create a function mock. This function allows to change behavior depending on whether we are +/// currently running a KUnit test or not. +/// +/// # Examples +/// +/// This example shows how a function can be mocked to return a well-known value while testing: +/// +/// ``` +/// # use kernel::kunit::in_kunit_test; +/// fn fn_mock_example(n: i32) -> i32 { +/// if in_kunit_test() { +/// return 100; +/// } +/// +/// n + 1 +/// } +/// +/// let mock_res = fn_mock_example(5); +/// assert_eq!(mock_res, 100); +/// ``` +pub fn in_kunit_test() -> bool { + // SAFETY: `kunit_get_current_test()` is always safe to call (it has fallbacks for + // when KUnit is not enabled). + !unsafe { bindings::kunit_get_current_test() }.is_null() +} + +#[cfg(CONFIG_RUST_KUNIT_SELFTEST)] +#[kunit_tests(rust_kernel_kunit)] +mod tests { + use super::*; + + #[test] + fn rust_test_kunit_example_test() { + assert_eq!(1 + 1, 2); + } + + #[test] + fn rust_test_kunit_in_kunit_test() { + assert!(in_kunit_test()); + } + + #[test] + #[cfg(not(all()))] + fn rust_test_kunit_always_disabled_test() { + // This test should never run because of the `cfg`. + assert!(false); + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs new file mode 100644 index 000000000..4d5c96ddc --- /dev/null +++ b/rust/kernel/lib.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` crate. +//! +//! This crate contains the kernel APIs that have been ported or wrapped for +//! usage by Rust code in the kernel and is shared by all of them. +//! +//! In other words, all the rest of the Rust code in the kernel (e.g. kernel +//! modules written in Rust) depends on [`core`] and this crate. +//! +//! If you need a kernel C API that is not ported or wrapped yet here, then +//! do so first instead of bypassing this crate. + +#![no_std] +// +// Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on +// the unstable features in use. +// +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// +// Stable since Rust 1.89.0. +#![feature(generic_arg_infer)] +// +// Expected to become stable. +#![feature(arbitrary_self_types)] +#![feature(derive_coerce_pointee)] +// +// To be determined. +#![feature(used_with_arg)] +// +// `feature(file_with_nul)` is stable since Rust 1.92.0. Before Rust 1.89.0, it did not exist, so +// enable it conditionally. +#![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))] + +// Ensure conditional compilation based on the kernel configuration works; +// otherwise we may silently break things like initcall handling. +#[cfg(not(CONFIG_RUST))] +compile_error!("Missing kernel configuration for conditional compilation"); + +// Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). +extern crate self as kernel; + +pub use ffi; + +pub mod acpi; +pub mod alloc; +#[cfg(CONFIG_AUXILIARY_BUS)] +pub mod auxiliary; +pub mod bitfield; +pub mod bitmap; +pub mod bits; +#[cfg(CONFIG_BLOCK)] +pub mod block; +pub mod bug; +pub mod build_assert; +pub mod clk; +#[cfg(CONFIG_CONFIGFS_FS)] +pub mod configfs; +pub mod cpu; +#[cfg(CONFIG_CPU_FREQ)] +pub mod cpufreq; +pub mod cpumask; +pub mod cred; +pub mod debugfs; +pub mod device; +pub mod device_id; +pub mod devres; +pub mod dma; +pub mod driver; +#[cfg(CONFIG_DRM = "y")] +pub mod drm; +pub mod error; +pub mod faux; +#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] +pub mod firmware; +pub mod fmt; +pub mod fs; +#[cfg(CONFIG_RUST_FWCTL_ABSTRACTIONS)] +pub mod fwctl; +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod gpu; +#[cfg(CONFIG_I2C = "y")] +pub mod i2c; +pub mod id_pool; +#[doc(hidden)] +pub mod impl_flags; +pub mod init; +pub mod interop; +pub mod interrupt; +pub mod io; +pub mod ioctl; +pub mod iommu; +pub mod iov; +pub mod irq; +pub mod jump_label; +#[cfg(CONFIG_KUNIT)] +pub mod kunit; +pub mod list; +pub mod maple_tree; +pub mod miscdevice; +pub mod mm; +pub mod module; +pub mod module_param; +#[cfg(CONFIG_NET)] +pub mod net; +pub mod num; +pub mod of; +#[cfg(CONFIG_PM_OPP)] +pub mod opp; +pub mod page; +#[cfg(CONFIG_PCI)] +pub mod pci; +pub mod pid_namespace; +pub mod platform; +pub mod prelude; +pub mod print; +pub mod processor; +pub mod ptr; +#[cfg(CONFIG_RUST_PWM_ABSTRACTIONS)] +pub mod pwm; +pub mod rbtree; +pub mod regulator; +pub mod revocable; +pub mod safety; +pub mod scatterlist; +pub mod security; +pub mod seq_file; +#[cfg(CONFIG_RUST_SERIAL_DEV_BUS_ABSTRACTIONS)] +pub mod serdev; +pub mod sizes; +#[cfg(CONFIG_SOC_BUS)] +pub mod soc; +#[doc(hidden)] +pub mod std_vendor; +pub mod str; +pub mod sync; +pub mod task; +pub mod time; +pub mod tracepoint; +pub mod transmute; +pub mod types; +pub mod uaccess; +#[cfg(CONFIG_USB = "y")] +pub mod usb; +pub mod workqueue; +pub mod xarray; + +#[doc(hidden)] +pub use bindings; +pub use macros; +pub use module::{ + InPlaceModule, + Module, + ModuleMetadata, + ThisModule, // +}; +pub use uapi; + +/// Prefix to appear before log messages printed from within the `kernel` crate. +const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; + +/// Dummy module type for `#[vtable]` `impl` blocks within the `kernel` crate (e.g. KUnit tests). +// The `allow` is needed since it may be unused (e.g. KUnit tests may be disabled). +#[allow(dead_code)] +struct LocalModule; + +impl ModuleMetadata for LocalModule { + const NAME: &'static str::CStr = c"rust_kernel"; + + const THIS_MODULE: ThisModule = { + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe { ThisModule::from_ptr(core::ptr::null_mut()) } + }; +} + +#[cfg(not(testlib))] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + pr_emerg!("{}\n", info); + // SAFETY: FFI call. + unsafe { bindings::BUG() }; +} + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or +/// [`Opaque::cast_from`] to resolve the mismatch. +/// +/// [`Opaque`]: crate::types::Opaque +/// [`Opaque::cast_into`]: crate::types::Opaque::cast_into +/// [`Opaque::cast_from`]: crate::types::Opaque::cast_from +/// +/// # Safety +/// +/// The pointer passed to this macro, and the pointer returned by this macro, must both be in +/// bounds of the same allocation. +/// +/// # Examples +/// +/// ``` +/// # use kernel::container_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr: *const _ = &test.b; +/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be +/// // in-bounds of the same allocation as `b_ptr`. +/// let test_alias = unsafe { container_of!(b_ptr, Test, b) }; +/// assert!(core::ptr::eq(&test, test_alias)); +/// ``` +#[macro_export] +macro_rules! container_of { + ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{ + let offset: usize = ::core::mem::offset_of!($Container, $($fields)*); + let field_ptr = $field_ptr; + let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>(); + $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut()); + container_ptr + }} +} + +/// Helper for [`container_of!`]. +#[doc(hidden)] +pub fn assert_same_type(_: T, _: T) {} + +/// Helper for `.rs.S` files. +#[doc(hidden)] +#[macro_export] +macro_rules! concat_literals { + ($( $asm:literal )* ) => { + ::core::concat!($($asm),*) + }; +} + +/// Wrapper around `asm!` configured for use in the kernel. +/// +/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!` +/// syntax. +// For x86, `asm!` uses intel syntax by default, but we want to use at&t syntax in the kernel. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[macro_export] +macro_rules! asm { + ($($asm:expr),* ; $($rest:tt)*) => { + ::core::arch::asm!( $($asm)*, options(att_syntax), $($rest)* ) + }; +} + +/// Wrapper around `asm!` configured for use in the kernel. +/// +/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!` +/// syntax. +// For non-x86 arches we just pass through to `asm!`. +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] +#[macro_export] +macro_rules! asm { + ($($asm:expr),* ; $($rest:tt)*) => { + ::core::arch::asm!( $($asm)*, $($rest)* ) + }; +} + +/// Gets the C string file name of a [`Location`]. +/// +/// If `Location::file_as_c_str()` is not available, returns a string that warns about it. +/// +/// [`Location`]: core::panic::Location +/// +/// # Examples +/// +/// ``` +/// # use kernel::file_from_location; +/// +/// #[track_caller] +/// fn foo() { +/// let caller = core::panic::Location::caller(); +/// +/// // Output: +/// // - A path like "rust/kernel/example.rs" if `file_as_c_str()` is available. +/// // - "" otherwise. +/// let caller_file = file_from_location(caller); +/// +/// // Prints out the message with caller's file name. +/// pr_info!("foo() called in file {caller_file:?}\n"); +/// +/// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) { +/// # assert_eq!(Ok(caller.file()), caller_file.to_str()); +/// # } +/// } +/// +/// # foo(); +/// ``` +#[inline] +pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr { + #[cfg(CONFIG_RUSTC_HAS_FILE_AS_C_STR)] + { + loc.file_as_c_str() + } + + #[cfg(all(CONFIG_RUSTC_HAS_FILE_WITH_NUL, not(CONFIG_RUSTC_HAS_FILE_AS_C_STR)))] + { + loc.file_with_nul() + } + + #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))] + { + let _ = loc; + c"" + } +} diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs new file mode 100644 index 000000000..0f367264e --- /dev/null +++ b/rust/kernel/list.rs @@ -0,0 +1,1222 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A linked list implementation. + +use crate::sync::ArcBorrow; +use crate::types::Opaque; +use core::iter::{DoubleEndedIterator, FusedIterator}; +use core::marker::PhantomData; +use core::ptr; +use pin_init::PinInit; + +mod impl_list_item_mod; +#[doc(inline)] +pub use self::impl_list_item_mod::{ + impl_has_list_links, + impl_has_list_links_self_ptr, + impl_list_item, + HasListLinks, + HasSelfPtr, // +}; + +mod arc; +#[doc(inline)] +pub use self::arc::{ + impl_list_arc_safe, + AtomicTracker, + ListArc, + ListArcSafe, + TryNewListArc, // +}; + +mod arc_field; +#[doc(inline)] +pub use self::arc_field::{ + define_list_arc_field_getter, + ListArcField, // +}; + +/// A linked list. +/// +/// All elements in this linked list will be [`ListArc`] references to the value. Since a value can +/// only have one `ListArc` (for each pair of prev/next pointers), this ensures that the same +/// prev/next pointers are not used for several linked lists. +/// +/// # Invariants +/// +/// * If the list is empty, then `first` is null. Otherwise, `first` points at the `ListLinks` +/// field of the first element in the list. +/// * All prev/next pointers in `ListLinks` fields of items in the list are valid and form a cycle. +/// * For every item in the list, the list owns the associated [`ListArc`] reference and has +/// exclusive access to the `ListLinks` field. +/// +/// # Examples +/// +/// Use [`ListLinks`] as the type of the intrusive field. +/// +/// ``` +/// use kernel::list::*; +/// +/// #[pin_data] +/// struct BasicItem { +/// value: i32, +/// #[pin] +/// links: ListLinks, +/// } +/// +/// impl BasicItem { +/// fn new(value: i32) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinks::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// impl_list_arc_safe! { +/// impl ListArcSafe<0> for BasicItem { untracked; } +/// } +/// impl_list_item! { +/// impl ListItem<0> for BasicItem { using ListLinks { self.links }; } +/// } +/// +/// // Create a new empty list. +/// let mut list = List::new(); +/// { +/// assert!(list.is_empty()); +/// } +/// +/// // Insert 3 elements using `push_back()`. +/// list.push_back(BasicItem::new(15)?); +/// list.push_back(BasicItem::new(10)?); +/// list.push_back(BasicItem::new(30)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [15, 10, 30] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 10); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 15); +/// } +/// +/// // Insert 3 elements using `push_front()`. +/// list.push_front(BasicItem::new(15)?); +/// list.push_front(BasicItem::new(10)?); +/// list.push_front(BasicItem::new(30)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [30, 10, 15] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_front()` and verify the content. +/// { +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 10); +/// } +/// +/// // Push `list2` to `list` through `push_all_back()`. +/// // list: [15] +/// // list2: [25, 35] +/// { +/// let mut list2 = List::new(); +/// list2.push_back(BasicItem::new(25)?); +/// list2.push_back(BasicItem::new(35)?); +/// +/// list.push_all_back(&mut list2); +/// +/// // list: [15, 25, 35] +/// // list2: [] +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 25); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 35); +/// assert!(iter.next().is_none()); +/// assert!(list2.is_empty()); +/// } +/// # Result::<(), Error>::Ok(()) +/// ``` +/// +/// Use [`ListLinksSelfPtr`] as the type of the intrusive field. This allows a list of trait object +/// type. +/// +/// ``` +/// use kernel::list::*; +/// +/// trait Foo { +/// fn foo(&self) -> (&'static str, i32); +/// } +/// +/// #[pin_data] +/// struct DTWrap { +/// #[pin] +/// links: ListLinksSelfPtr>, +/// value: T, +/// } +/// +/// impl DTWrap { +/// fn new(value: T) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinksSelfPtr::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// impl_list_arc_safe! { +/// impl{T: ?Sized} ListArcSafe<0> for DTWrap { untracked; } +/// } +/// impl_list_item! { +/// impl ListItem<0> for DTWrap { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// // Create a new empty list. +/// let mut list = List::>::new(); +/// { +/// assert!(list.is_empty()); +/// } +/// +/// struct A(i32); +/// // `A` returns the inner value for `foo`. +/// impl Foo for A { fn foo(&self) -> (&'static str, i32) { ("a", self.0) } } +/// +/// struct B; +/// // `B` always returns 42. +/// impl Foo for B { fn foo(&self) -> (&'static str, i32) { ("b", 42) } } +/// +/// // Insert 3 element using `push_back()`. +/// list.push_back(DTWrap::new(A(15))?); +/// list.push_back(DTWrap::new(A(32))?); +/// list.push_back(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [A(15), A(32), B] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// } +/// +/// // Insert 3 elements using `push_front()`. +/// list.push_front(DTWrap::new(A(15))?); +/// list.push_front(DTWrap::new(A(32))?); +/// list.push_front(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [B, A(32), A(15)] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// } +/// +/// // Push `list2` to `list` through `push_all_back()`. +/// // list: [B] +/// // list2: [B, A(25)] +/// { +/// let mut list2 = List::>::new(); +/// list2.push_back(DTWrap::new(B)?); +/// list2.push_back(DTWrap::new(A(25))?); +/// +/// list.push_all_back(&mut list2); +/// +/// // list: [B, B, A(25)] +/// // list2: [] +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 25)); +/// assert!(iter.next().is_none()); +/// assert!(list2.is_empty()); +/// } +/// # Result::<(), Error>::Ok(()) +/// ``` +pub struct List, const ID: u64 = 0> { + first: *mut ListLinksFields, + _ty: PhantomData>, +} + +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Send for List +where + ListArc: Send, + T: ?Sized + ListItem, +{ +} +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Sync for List +where + ListArc: Sync, + T: ?Sized + ListItem, +{ +} + +/// Implemented by types where a [`ListArc`] can be inserted into a [`List`]. +/// +/// # Safety +/// +/// Implementers must ensure that they provide the guarantees documented on methods provided by +/// this trait. +/// +/// [`ListArc`]: ListArc +pub unsafe trait ListItem: ListArcSafe { + /// Views the [`ListLinks`] for this value. + /// + /// # Guarantees + /// + /// If there is a previous call to `prepare_to_insert` and there is no call to `post_remove` + /// since the most recent such call, then this returns the same pointer as the one returned by + /// the most recent call to `prepare_to_insert`. + /// + /// Otherwise, the returned pointer points at a read-only [`ListLinks`] with two null pointers. + /// + /// # Safety + /// + /// The provided pointer must point at a valid value. (It need not be in an `Arc`.) + unsafe fn view_links(me: *const Self) -> *mut ListLinks; + + /// View the full value given its [`ListLinks`] field. + /// + /// Can only be used when the value is in a list. + /// + /// # Guarantees + /// + /// * Returns the same pointer as the one passed to the most recent call to `prepare_to_insert`. + /// * The returned pointer is valid until the next call to `post_remove`. + /// + /// # Safety + /// + /// * The provided pointer must originate from the most recent call to `prepare_to_insert`, or + /// from a call to `view_links` that happened after the most recent call to + /// `prepare_to_insert`. + /// * Since the most recent call to `prepare_to_insert`, the `post_remove` method must not have + /// been called. + unsafe fn view_value(me: *mut ListLinks) -> *const Self; + + /// This is called when an item is inserted into a [`List`]. + /// + /// # Guarantees + /// + /// The caller is granted exclusive access to the returned [`ListLinks`] until `post_remove` is + /// called. + /// + /// # Safety + /// + /// * The provided pointer must point at a valid value in an [`Arc`]. + /// * Calls to `prepare_to_insert` and `post_remove` on the same value must alternate. + /// * The caller must own the [`ListArc`] for this value. + /// * The caller must not give up ownership of the [`ListArc`] unless `post_remove` has been + /// called after this call to `prepare_to_insert`. + /// + /// [`Arc`]: crate::sync::Arc + unsafe fn prepare_to_insert(me: *const Self) -> *mut ListLinks; + + /// This undoes a previous call to `prepare_to_insert`. + /// + /// # Guarantees + /// + /// The returned pointer is the pointer that was originally passed to `prepare_to_insert`. + /// + /// # Safety + /// + /// The provided pointer must be the pointer returned by the most recent call to + /// `prepare_to_insert`. + unsafe fn post_remove(me: *mut ListLinks) -> *const Self; +} + +#[repr(C)] +#[derive(Copy, Clone)] +struct ListLinksFields { + next: *mut ListLinksFields, + prev: *mut ListLinksFields, +} + +/// The prev/next pointers for an item in a linked list. +/// +/// # Invariants +/// +/// The fields are null if and only if this item is not in a list. +#[repr(transparent)] +pub struct ListLinks { + // This type is `!Unpin` for aliasing reasons as the pointers are part of an intrusive linked + // list. + inner: Opaque, +} + +// SAFETY: The only way to access/modify the pointers inside of `ListLinks` is via holding the +// associated `ListArc`. Since that type correctly implements `Send`, it is impossible to +// move this an instance of this type to a different thread if the pointees are `!Send`. +unsafe impl Send for ListLinks {} +// SAFETY: The type is opaque so immutable references to a ListLinks are useless. Therefore, it's +// okay to have immutable access to a ListLinks from several threads at once. +unsafe impl Sync for ListLinks {} + +impl ListLinks { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + } + } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn fields(me: *mut Self) -> *mut ListLinksFields { + // SAFETY: The caller promises that the pointer is valid. + unsafe { Opaque::cast_into(ptr::addr_of!((*me).inner)) } + } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn from_fields(me: *mut ListLinksFields) -> *mut Self { + me.cast() + } +} + +/// Similar to [`ListLinks`], but also contains a pointer to the full value. +/// +/// This type can be used instead of [`ListLinks`] to support lists with trait objects. +#[repr(C)] +pub struct ListLinksSelfPtr { + /// The `ListLinks` field inside this value. + /// + /// This is public so that it can be used with `impl_has_list_links!`. + pub inner: ListLinks, + // UnsafeCell is not enough here because we use `Opaque::uninit` as a dummy value, and + // `ptr::null()` doesn't work for `T: ?Sized`. + self_ptr: Opaque<*const T>, +} + +// SAFETY: The fields of a ListLinksSelfPtr can be moved across thread boundaries. +unsafe impl Send for ListLinksSelfPtr {} +// SAFETY: The type is opaque so immutable references to a ListLinksSelfPtr are useless. Therefore, +// it's okay to have immutable access to a ListLinks from several threads at once. +// +// Note that `inner` being a public field does not prevent this type from being opaque, since +// `inner` is a opaque type. +unsafe impl Sync for ListLinksSelfPtr {} + +impl ListLinksSelfPtr { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + }, + self_ptr: Opaque::uninit(), + } + } + + /// Returns a pointer to the self pointer. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + pub unsafe fn raw_get_self_ptr(me: *const Self) -> *const Opaque<*const T> { + // SAFETY: The caller promises that the pointer is valid. + unsafe { ptr::addr_of!((*me).self_ptr) } + } +} + +impl, const ID: u64> List { + /// Creates a new empty list. + pub const fn new() -> Self { + Self { + first: ptr::null_mut(), + _ty: PhantomData, + } + } + + /// Returns whether this list is empty. + pub fn is_empty(&self) -> bool { + self.first.is_null() + } + + /// Inserts `item` before `next` in the cycle. + /// + /// Returns a pointer to the newly inserted element. Never changes `self.first` unless the list + /// is empty. + /// + /// # Safety + /// + /// * `next` must be an element in this list or null. + /// * if `next` is null, then the list must be empty. + unsafe fn insert_inner( + &mut self, + item: ListArc, + next: *mut ListLinksFields, + ) -> *mut ListLinksFields { + let raw_item = ListArc::into_raw(item); + // SAFETY: + // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. + // * Since we have ownership of the `ListArc`, `post_remove` must have been called after + // the most recent call to `prepare_to_insert`, if any. + // * We own the `ListArc`. + // * Removing items from this list is always done using `remove_internal_inner`, which + // calls `post_remove` before giving up ownership. + let list_links = unsafe { T::prepare_to_insert(raw_item) }; + // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. + let item = unsafe { ListLinks::fields(list_links) }; + + // Check if the list is empty. + if next.is_null() { + // SAFETY: The caller just gave us ownership of these fields. + // INVARIANT: A linked list with one item should be cyclic. + unsafe { + (*item).next = item; + (*item).prev = item; + } + self.first = item; + } else { + // SAFETY: By the type invariant, this pointer is valid or null. We just checked that + // it's not null, so it must be valid. + let prev = unsafe { (*next).prev }; + // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us + // ownership of the fields on `item`. + // INVARIANT: This correctly inserts `item` between `prev` and `next`. + unsafe { + (*item).next = next; + (*item).prev = prev; + (*prev).next = item; + (*next).prev = item; + } + } + + item + } + + /// Add the provided item to the back of the list. + pub fn push_back(&mut self, item: ListArc) { + // SAFETY: + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + unsafe { self.insert_inner(item, self.first) }; + } + + /// Add the provided item to the front of the list. + pub fn push_front(&mut self, item: ListArc) { + // SAFETY: + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + let new_elem = unsafe { self.insert_inner(item, self.first) }; + + // INVARIANT: `new_elem` is in the list because we just inserted it. + self.first = new_elem; + } + + /// Removes the last item from this list. + pub fn pop_back(&mut self) -> Option> { + if self.is_empty() { + return None; + } + + // SAFETY: We just checked that the list is not empty. + let last = unsafe { (*self.first).prev }; + // SAFETY: The last item of this list is in this list. + Some(unsafe { self.remove_internal(last) }) + } + + /// Removes the first item from this list. + pub fn pop_front(&mut self) -> Option> { + if self.is_empty() { + return None; + } + + // SAFETY: The first item of this list is in this list. + Some(unsafe { self.remove_internal(self.first) }) + } + + /// Removes the provided item from this list and returns it. + /// + /// This returns `None` if the item is not in the list. (Note that by the safety requirements, + /// this means that the item is not in any list.) + /// + /// When using this method, be careful with using `mem::take` on the same list as that may + /// result in violating the safety requirements of this method. + /// + /// # Safety + /// + /// `item` must not be in a different linked list (with the same id). + pub unsafe fn remove(&mut self, item: &T) -> Option> { + // SAFETY: TODO. + let mut item = unsafe { ListLinks::fields(T::view_links(item)) }; + // SAFETY: The user provided a reference, and reference are never dangling. + // + // As for why this is not a data race, there are two cases: + // + // * If `item` is not in any list, then these fields are read-only and null. + // * If `item` is in this list, then we have exclusive access to these fields since we + // have a mutable reference to the list. + // + // In either case, there's no race. + let ListLinksFields { next, prev } = unsafe { *item }; + + debug_assert_eq!(next.is_null(), prev.is_null()); + if !next.is_null() { + // This is really a no-op, but this ensures that `item` is a raw pointer that was + // obtained without going through a pointer->reference->pointer conversion roundtrip. + // This ensures that the list is valid under the more restrictive strict provenance + // ruleset. + // + // SAFETY: We just checked that `next` is not null, and it's not dangling by the + // list invariants. + unsafe { + debug_assert_eq!(item, (*next).prev); + item = (*next).prev; + } + + // SAFETY: We just checked that `item` is in a list, so the caller guarantees that it + // is in this list. The pointers are in the right order. + Some(unsafe { self.remove_internal_inner(item, next, prev) }) + } else { + None + } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// `item` must point at an item in this list. + unsafe fn remove_internal(&mut self, item: *mut ListLinksFields) -> ListArc { + // SAFETY: The caller promises that this pointer is not dangling, and there's no data race + // since we have a mutable reference to the list containing `item`. + let ListLinksFields { next, prev } = unsafe { *item }; + // SAFETY: The pointers are ok and in the right order. + unsafe { self.remove_internal_inner(item, next, prev) } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// The `item` pointer must point at an item in this list, and we must have `(*item).next == + /// next` and `(*item).prev == prev`. + unsafe fn remove_internal_inner( + &mut self, + item: *mut ListLinksFields, + next: *mut ListLinksFields, + prev: *mut ListLinksFields, + ) -> ListArc { + // SAFETY: We have exclusive access to the pointers of items in the list, and the prev/next + // pointers are always valid for items in a list. + // + // INVARIANT: There are three cases: + // * If the list has at least three items, then after removing the item, `prev` and `next` + // will be next to each other. + // * If the list has two items, then the remaining item will point at itself. + // * If the list has one item, then `next == prev == item`, so these writes have no + // effect. The list remains unchanged and `item` is still in the list for now. + unsafe { + (*next).prev = prev; + (*prev).next = next; + } + // SAFETY: We have exclusive access to items in the list. + // INVARIANT: `item` is being removed, so the pointers should be null. + unsafe { + (*item).prev = ptr::null_mut(); + (*item).next = ptr::null_mut(); + } + // INVARIANT: There are three cases: + // * If `item` was not the first item, then `self.first` should remain unchanged. + // * If `item` was the first item and there is another item, then we just updated + // `prev->next` to `next`, which is the new first item, and setting `item->next` to null + // did not modify `prev->next`. + // * If `item` was the only item in the list, then `prev == item`, and we just set + // `item->next` to null, so this correctly sets `first` to null now that the list is + // empty. + if self.first == item { + // SAFETY: The `prev` pointer is the value that `item->prev` had when it was in this + // list, so it must be valid. There is no race since `prev` is still in the list and we + // still have exclusive access to the list. + self.first = unsafe { (*prev).next }; + } + + // SAFETY: `item` used to be in the list, so it is dereferenceable by the type invariants + // of `List`. + let list_links = unsafe { ListLinks::from_fields(item) }; + // SAFETY: Any pointer in the list originates from a `prepare_to_insert` call. + let raw_item = unsafe { T::post_remove(list_links) }; + // SAFETY: The above call to `post_remove` guarantees that we can recreate the `ListArc`. + unsafe { ListArc::from_raw(raw_item) } + } + + /// Moves all items from `other` into `self`. + /// + /// The items of `other` are added to the back of `self`, so the last item of `other` becomes + /// the last item of `self`. + pub fn push_all_back(&mut self, other: &mut List) { + // First, we insert the elements into `self`. At the end, we make `other` empty. + if self.is_empty() { + // INVARIANT: All of the elements in `other` become elements of `self`. + self.first = other.first; + } else if !other.is_empty() { + let other_first = other.first; + // SAFETY: The other list is not empty, so this pointer is valid. + let other_last = unsafe { (*other_first).prev }; + let self_first = self.first; + // SAFETY: The self list is not empty, so this pointer is valid. + let self_last = unsafe { (*self_first).prev }; + + // SAFETY: We have exclusive access to both lists, so we can update the pointers. + // INVARIANT: This correctly sets the pointers to merge both lists. We do not need to + // update `self.first` because the first element of `self` does not change. + unsafe { + (*self_first).prev = other_last; + (*other_last).next = self_first; + (*self_last).next = other_first; + (*other_first).prev = self_last; + } + } + + // INVARIANT: The other list is now empty, so update its pointer. + other.first = ptr::null_mut(); + } + + /// Returns a cursor that points before the first element of the list. + pub fn cursor_front(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `self.first` is in this list. + Cursor { + next: self.first, + list: self, + } + } + + /// Returns a cursor that points after the last element in the list. + pub fn cursor_back(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `next` is allowed to be null. + Cursor { + next: core::ptr::null_mut(), + list: self, + } + } + + /// Creates an iterator over the list. + pub fn iter(&self) -> Iter<'_, T, ID> { + // INVARIANT: If the list is empty, both pointers are null. Otherwise, both pointers point + // at the first element of the same list. + Iter { + current: self.first, + stop: self.first, + _ty: PhantomData, + } + } +} + +impl, const ID: u64> Default for List { + fn default() -> Self { + List::new() + } +} + +impl, const ID: u64> Drop for List { + fn drop(&mut self) { + while let Some(item) = self.pop_front() { + drop(item); + } + } +} + +/// An iterator over a [`List`]. +/// +/// # Invariants +/// +/// * There must be a [`List`] that is immutably borrowed for the duration of `'a`. +/// * The `current` pointer is null or points at a value in that [`List`]. +/// * The `stop` pointer is equal to the `first` field of that [`List`]. +#[derive(Clone)] +pub struct Iter<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + current: *mut ListLinksFields, + stop: *mut ListLinksFields, + _ty: PhantomData<&'a ListArc>, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Iterator for Iter<'a, T, ID> { + type Item = ArcBorrow<'a, T>; + + fn next(&mut self) -> Option> { + if self.current.is_null() { + return None; + } + + let current = self.current; + + // SAFETY: We just checked that `current` is not null, so it is in a list, and hence not + // dangling. There's no race because the iterator holds an immutable borrow to the list. + let next = unsafe { (*current).next }; + // INVARIANT: If `current` was the last element of the list, then this updates it to null. + // Otherwise, we update it to the next element. + self.current = if next != self.stop { + next + } else { + ptr::null_mut() + }; + + // SAFETY: The `current` pointer points at a value in the list. + let item = unsafe { T::view_value(ListLinks::from_fields(current)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the list. However, the `ArcBorrow` is annotated with the iterator's + // lifetime, and the list is immutably borrowed for that lifetime. + // * Values in a list never have a `UniqueArc` reference. + Some(unsafe { ArcBorrow::from_raw(item) }) + } +} + +/// A cursor into a [`List`]. +/// +/// A cursor always rests between two elements in the list. This means that a cursor has a previous +/// and next element, but no current element. It also means that it's possible to have a cursor +/// into an empty list. +/// +/// # Examples +/// +/// ``` +/// use kernel::prelude::*; +/// use kernel::list::{List, ListArc, ListLinks}; +/// +/// #[pin_data] +/// struct ListItem { +/// value: u32, +/// #[pin] +/// links: ListLinks, +/// } +/// +/// impl ListItem { +/// fn new(value: u32) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinks::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for ListItem { untracked; } +/// } +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for ListItem { using ListLinks { self.links }; } +/// } +/// +/// // Use a cursor to remove the first element with the given value. +/// fn remove_first(list: &mut List, value: u32) -> Option> { +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// return Some(next.remove()); +/// } +/// cursor.move_next(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove the last element with the given value. +/// fn remove_last(list: &mut List, value: u32) -> Option> { +/// let mut cursor = list.cursor_back(); +/// while let Some(prev) = cursor.peek_prev() { +/// if prev.value == value { +/// return Some(prev.remove()); +/// } +/// cursor.move_prev(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove all elements with the given value. The removed elements are moved to +/// // a new list. +/// fn remove_all(list: &mut List, value: u32) -> List { +/// let mut out = List::new(); +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// out.push_back(next.remove()); +/// } else { +/// cursor.move_next(); +/// } +/// } +/// out +/// } +/// +/// // Use a cursor to insert a value at a specific index. Returns an error if the index is out of +/// // bounds. +/// fn insert_at(list: &mut List, new: ListArc, idx: usize) -> Result { +/// let mut cursor = list.cursor_front(); +/// for _ in 0..idx { +/// if !cursor.move_next() { +/// return Err(EINVAL); +/// } +/// } +/// cursor.insert_next(new); +/// Ok(()) +/// } +/// +/// // Merge two sorted lists into a single sorted list. +/// fn merge_sorted(list: &mut List, merge: List) { +/// let mut cursor = list.cursor_front(); +/// for to_insert in merge { +/// while let Some(next) = cursor.peek_next() { +/// if to_insert.value < next.value { +/// break; +/// } +/// cursor.move_next(); +/// } +/// cursor.insert_prev(to_insert); +/// } +/// } +/// +/// let mut list = List::new(); +/// list.push_back(ListItem::new(14)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(10)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(15)?); +/// list.push_back(ListItem::new(14)?); +/// assert_eq!(remove_all(&mut list, 12).iter().count(), 2); +/// // [14, 10, 15, 14] +/// assert!(remove_first(&mut list, 14).is_some()); +/// // [10, 15, 14] +/// insert_at(&mut list, ListItem::new(12)?, 2)?; +/// // [10, 15, 12, 14] +/// assert!(remove_last(&mut list, 15).is_some()); +/// // [10, 12, 14] +/// +/// let mut list2 = List::new(); +/// list2.push_back(ListItem::new(11)?); +/// list2.push_back(ListItem::new(13)?); +/// merge_sorted(&mut list, list2); +/// +/// let mut items = list.into_iter(); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 11); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 12); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 13); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 14); +/// assert!(items.next().is_none()); +/// # Result::<(), Error>::Ok(()) +/// ``` +/// +/// # Invariants +/// +/// The `next` pointer is null or points a value in `list`. +pub struct Cursor<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + list: &'a mut List, + /// Points at the element after this cursor, or null if the cursor is after the last element. + next: *mut ListLinksFields, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Cursor<'a, T, ID> { + /// Returns a pointer to the element before the cursor. + /// + /// Returns null if there is no element before the cursor. + fn prev_ptr(&self) -> *mut ListLinksFields { + let mut next = self.next; + let first = self.list.first; + if next == first { + // We are before the first element. + return core::ptr::null_mut(); + } + + if next.is_null() { + // We are after the last element, so we need a pointer to the last element, which is + // the same as `(*first).prev`. + next = first; + } + + // SAFETY: `next` can't be null, because then `first` must also be null, but in that case + // we would have exited at the `next == first` check. Thus, `next` is an element in the + // list, so we can access its `prev` pointer. + unsafe { (*next).prev } + } + + /// Access the element after this cursor. + pub fn peek_next(&mut self) -> Option> { + if self.next.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `self.next` is non-null, so it must be in `self.list`. + // * `ptr` is equal to `self.next`. + Some(CursorPeek { + ptr: self.next, + cursor: self, + }) + } + + /// Access the element before this cursor. + pub fn peek_prev(&mut self) -> Option> { + let prev = self.prev_ptr(); + + if prev.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `prev` is non-null, so it must be in `self.list`. + // * `self.prev_ptr()` never returns `self.next`. + Some(CursorPeek { + ptr: prev, + cursor: self, + }) + } + + /// Move the cursor one element forward. + /// + /// If the cursor is after the last element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_next(&mut self) -> bool { + if self.next.is_null() { + return false; + } + + // SAFETY: `self.next` is an element in the list and we borrow the list mutably, so we can + // access the `next` field. + let mut next = unsafe { (*self.next).next }; + + if next == self.list.first { + next = core::ptr::null_mut(); + } + + // INVARIANT: `next` is either null or the next element after an element in the list. + self.next = next; + true + } + + /// Move the cursor one element backwards. + /// + /// If the cursor is before the first element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_prev(&mut self) -> bool { + if self.next == self.list.first { + return false; + } + + // INVARIANT: `prev_ptr()` always returns a pointer that is null or in the list. + self.next = self.prev_ptr(); + true + } + + /// Inserts an element where the cursor is pointing and get a pointer to the new element. + fn insert_inner(&mut self, item: ListArc) -> *mut ListLinksFields { + let ptr = if self.next.is_null() { + self.list.first + } else { + self.next + }; + // SAFETY: + // * `ptr` is an element in the list or null. + // * if `ptr` is null, then `self.list.first` is null so the list is empty. + let item = unsafe { self.list.insert_inner(item, ptr) }; + if self.next == self.list.first { + // INVARIANT: We just inserted `item`, so it's a member of list. + self.list.first = item; + } + item + } + + /// Insert an element at this cursor's location. + pub fn insert(mut self, item: ListArc) { + // This is identical to `insert_prev`, but consumes the cursor. This is helpful because it + // reduces confusion when the last operation on the cursor is an insertion; in that case, + // you just want to insert the element at the cursor, and it is confusing that the call + // involves the word prev or next. + self.insert_inner(item); + } + + /// Inserts an element after this cursor. + /// + /// After insertion, the new element will be after the cursor. + pub fn insert_next(&mut self, item: ListArc) { + self.next = self.insert_inner(item); + } + + /// Inserts an element before this cursor. + /// + /// After insertion, the new element will be before the cursor. + pub fn insert_prev(&mut self, item: ListArc) { + self.insert_inner(item); + } + + /// Remove the next element from the list. + pub fn remove_next(&mut self) -> Option> { + self.peek_next().map(|v| v.remove()) + } + + /// Remove the previous element from the list. + pub fn remove_prev(&mut self) -> Option> { + self.peek_prev().map(|v| v.remove()) + } +} + +/// References the element in the list next to the cursor. +/// +/// # Invariants +/// +/// * `ptr` is an element in `self.cursor.list`. +/// * `ISNEXT == (self.ptr == self.cursor.next)`. +pub struct CursorPeek<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> { + cursor: &'a mut Cursor<'b, T, ID>, + ptr: *mut ListLinksFields, +} + +impl<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> + CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + /// Remove the element from the list. + pub fn remove(self) -> ListArc { + if ISNEXT { + self.cursor.move_next(); + } + + // INVARIANT: `self.ptr` is not equal to `self.cursor.next` due to the above `move_next` + // call. + // SAFETY: By the type invariants of `Self`, `next` is not null, so `next` is an element of + // `self.cursor.list` by the type invariants of `Cursor`. + unsafe { self.cursor.list.remove_internal(self.ptr) } + } + + /// Access this value as an [`ArcBorrow`]. + pub fn arc(&self) -> ArcBorrow<'_, T> { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `ArcBorrow` holds + // an immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable + // access requires first releasing the immutable borrow on the `CursorPeek`. + // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc` + // reference, and `UniqueArc` references must be unique. + unsafe { ArcBorrow::from_raw(me) } + } +} + +impl<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> core::ops::Deref + for CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + // If you change the `ptr` field to have type `ArcBorrow<'a, T>`, it might seem like you could + // get rid of the `CursorPeek::arc` method and change the deref target to `ArcBorrow<'a, T>`. + // However, that doesn't work because 'a is too long. You could obtain an `ArcBorrow<'a, T>` + // and then call `CursorPeek::remove` without giving up the `ArcBorrow<'a, T>`, which would be + // unsound. + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + + // SAFETY: The value cannot be removed from the list for the duration of the lifetime + // annotated on the returned `&T`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `&T` holds an + // immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable access + // requires first releasing the immutable borrow on the `CursorPeek`. + unsafe { &*me } + } +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> FusedIterator for Iter<'a, T, ID> {} + +impl<'a, T: ?Sized + ListItem, const ID: u64> IntoIterator for &'a List { + type IntoIter = Iter<'a, T, ID>; + type Item = ArcBorrow<'a, T>; + + fn into_iter(self) -> Iter<'a, T, ID> { + self.iter() + } +} + +/// An owning iterator into a [`List`]. +pub struct IntoIter, const ID: u64 = 0> { + list: List, +} + +impl, const ID: u64> Iterator for IntoIter { + type Item = ListArc; + + fn next(&mut self) -> Option> { + self.list.pop_front() + } +} + +impl, const ID: u64> FusedIterator for IntoIter {} + +impl, const ID: u64> DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option> { + self.list.pop_back() + } +} + +impl, const ID: u64> IntoIterator for List { + type IntoIter = IntoIter; + type Item = ListArc; + + fn into_iter(self) -> IntoIter { + IntoIter { list: self } + } +} diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs new file mode 100644 index 000000000..209b1173c --- /dev/null +++ b/rust/kernel/list/arc.rs @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A wrapper around `Arc` for linked lists. + +use crate::alloc::{AllocError, Flags}; +use crate::prelude::*; +use crate::sync::atomic::{ordering, AtomicFlag}; +use crate::sync::{Arc, ArcBorrow, UniqueArc}; +use core::marker::PhantomPinned; +use core::ops::Deref; +use core::pin::Pin; + +/// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for +/// this id. +/// +/// Types that implement this trait should include some kind of logic for keeping track of whether +/// a [`ListArc`] exists or not. We refer to this logic as "the tracking inside `T`". +/// +/// We allow the case where the tracking inside `T` thinks that a [`ListArc`] exists, but actually, +/// there isn't a [`ListArc`]. However, we do not allow the opposite situation where a [`ListArc`] +/// exists, but the tracking thinks it doesn't. This is because the former can at most result in us +/// failing to create a [`ListArc`] when the operation could succeed, whereas the latter can result +/// in the creation of two [`ListArc`] references. Only the latter situation can lead to memory +/// safety issues. +/// +/// A consequence of the above is that you may implement the tracking inside `T` by not actually +/// keeping track of anything. To do this, you always claim that a [`ListArc`] exists, even if +/// there isn't one. This implementation is allowed by the above rule, but it means that +/// [`ListArc`] references can only be created if you have ownership of *all* references to the +/// refcounted object, as you otherwise have no way of knowing whether a [`ListArc`] exists. +pub trait ListArcSafe { + /// Informs the tracking inside this type that it now has a [`ListArc`] reference. + /// + /// This method may be called even if the tracking inside this type thinks that a `ListArc` + /// reference exists. (But only if that's not actually the case.) + /// + /// # Safety + /// + /// Must not be called if a [`ListArc`] already exist for this value. + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>); + + /// Informs the tracking inside this type that there is no [`ListArc`] reference anymore. + /// + /// # Safety + /// + /// Must only be called if there is no [`ListArc`] reference, but the tracking thinks there is. + unsafe fn on_drop_list_arc(&self); +} + +/// Declares that this type is able to safely attempt to create `ListArc`s at any time. +/// +/// # Safety +/// +/// The guarantees of `try_new_list_arc` must be upheld. +pub unsafe trait TryNewListArc: ListArcSafe { + /// Attempts to convert an `Arc` into an `ListArc`. Returns `true` if the + /// conversion was successful. + /// + /// This method should not be called directly. Use [`ListArc::try_from_arc`] instead. + /// + /// # Guarantees + /// + /// If this call returns `true`, then there is no [`ListArc`] pointing to this value. + /// Additionally, this call will have transitioned the tracking inside `Self` from not thinking + /// that a [`ListArc`] exists, to thinking that a [`ListArc`] exists. + fn try_new_list_arc(&self) -> bool; +} + +/// Declares that this type supports [`ListArc`]. +/// +/// This macro supports a few different strategies for implementing the tracking inside the type: +/// +/// * The `untracked` strategy does not actually keep track of whether a [`ListArc`] exists. When +/// using this strategy, the only way to create a [`ListArc`] is using a [`UniqueArc`]. +/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user must specify +/// which field to defer the tracking to. The field must implement [`ListArcSafe`]. If the field +/// implements [`TryNewListArc`], then the type will also implement [`TryNewListArc`]. +/// +/// The `tracked_by` strategy is usually used by deferring to a field of type +/// [`AtomicTracker`]. However, it is also possible to defer the tracking to another struct +/// using also using this macro. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_list_arc_safe { + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {} + unsafe fn on_drop_list_arc(&self) {} + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { + tracked_by $field:ident : $fty:ty; + } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) { + ::pin_init::assert_pinned!($t, $field, $fty, inline); + + // SAFETY: This field is structurally pinned as per the above assertion. + let field = unsafe { + ::core::pin::Pin::map_unchecked_mut(self, |me| &mut me.$field) + }; + // SAFETY: The caller promises that there is no `ListArc`. + unsafe { + <$fty as $crate::list::ListArcSafe<$num>>::on_create_list_arc_from_unique(field) + }; + } + unsafe fn on_drop_list_arc(&self) { + // SAFETY: The caller promises that there is no `ListArc` reference, and also + // promises that the tracking thinks there is a `ListArc` reference. + unsafe { <$fty as $crate::list::ListArcSafe<$num>>::on_drop_list_arc(&self.$field) }; + } + } + unsafe impl$(<$($generics)*>)? $crate::list::TryNewListArc<$num> for $t + where + $fty: TryNewListArc<$num>, + { + fn try_new_list_arc(&self) -> bool { + <$fty as $crate::list::TryNewListArc<$num>>::try_new_list_arc(&self.$field) + } + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + + () => {}; +} +pub use impl_list_arc_safe; + +/// A wrapper around [`Arc`] that's guaranteed unique for the given id. +/// +/// The `ListArc` type can be thought of as a special reference to a refcounted object that owns the +/// permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring +/// that each object has only one `ListArc` reference, the owner of that reference is assured +/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a [`List`], +/// the [`List`] takes ownership of the `ListArc` reference. +/// +/// There are various strategies to ensuring that a value has only one `ListArc` reference. The +/// simplest is to convert a [`UniqueArc`] into a `ListArc`. However, the refcounted object could +/// also keep track of whether a `ListArc` exists using a boolean, which could allow for the +/// creation of new `ListArc` references from an [`Arc`] reference. Whatever strategy is used, the +/// relevant tracking is referred to as "the tracking inside `T`", and the [`ListArcSafe`] trait +/// (and its subtraits) are used to update the tracking when a `ListArc` is created or destroyed. +/// +/// Note that we allow the case where the tracking inside `T` thinks that a `ListArc` exists, but +/// actually, there isn't a `ListArc`. However, we do not allow the opposite situation where a +/// `ListArc` exists, but the tracking thinks it doesn't. This is because the former can at most +/// result in us failing to create a `ListArc` when the operation could succeed, whereas the latter +/// can result in the creation of two `ListArc` references. +/// +/// While this `ListArc` is unique for the given id, there still might exist normal `Arc` +/// references to the object. +/// +/// # Invariants +/// +/// * Each reference counted object has at most one `ListArc` for each value of `ID`. +/// * The tracking inside `T` is aware that a `ListArc` reference exists. +/// +/// [`List`]: crate::list::List +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct ListArc +where + T: ListArcSafe + ?Sized, +{ + arc: Arc, +} + +impl, const ID: u64> ListArc { + /// Constructs a new reference counted instance of `T`. + #[inline] + pub fn new(contents: T, flags: Flags) -> Result { + Ok(Self::from(UniqueArc::new(contents, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + // We don't implement `InPlaceInit` because `ListArc` is implicitly pinned. This is similar to + // what we do for `Arc`. + #[inline] + pub fn pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_pin_init(init, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// This is equivalent to [`ListArc::pin_init`], since a [`ListArc`] is always pinned. + #[inline] + pub fn init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_init(init, flags)?)) + } +} + +impl From> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(unique: UniqueArc) -> Self { + Self::from(Pin::from(unique)) + } +} + +impl From>> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a pinned [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(mut unique: Pin>) -> Self { + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { T::on_create_list_arc_from_unique(unique.as_mut()) }; + let arc = Arc::from(unique); + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc`, + // so we can create a `ListArc`. + unsafe { Self::transmute_from_arc(arc) } + } +} + +impl ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Creates two `ListArc`s from a [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_unique(unique: UniqueArc) -> (Self, ListArc) + where + T: ListArcSafe, + { + Self::pair_from_pin_unique(Pin::from(unique)) + } + + /// Creates two `ListArc`s from a pinned [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_pin_unique( + mut unique: Pin>, + ) -> (Self, ListArc) + where + T: ListArcSafe, + { + build_assert!(ID != ID2); + + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + + let arc1 = Arc::from(unique); + let arc2 = Arc::clone(&arc1); + + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc` + // for both IDs (which are different), so we can create two `ListArc`s. + unsafe { + ( + Self::transmute_from_arc(arc1), + ListArc::transmute_from_arc(arc2), + ) + } + } + + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc(arc: Arc) -> Result> + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Ok(unsafe { Self::transmute_from_arc(arc) }) + } else { + Err(arc) + } + } + + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc_borrow(arc: ArcBorrow<'_, T>) -> Option + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Some(unsafe { Self::transmute_from_arc(Arc::from(arc)) }) + } else { + None + } + } + + /// Try to create a new `ListArc`. + /// + /// If it's not possible to create a new `ListArc`, then the `Arc` is dropped. This will never + /// run the destructor of the value. + pub fn try_from_arc_or_drop(arc: Arc) -> Option + where + T: TryNewListArc, + { + match Self::try_from_arc(arc) { + Ok(list_arc) => Some(list_arc), + Err(arc) => Arc::into_unique_or_drop(arc).map(Self::from), + } + } + + /// Transmutes an [`Arc`] into a `ListArc` without updating the tracking inside `T`. + /// + /// # Safety + /// + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + unsafe fn transmute_from_arc(arc: Arc) -> Self { + // INVARIANT: By the safety requirements, the invariants on `ListArc` are satisfied. + Self { arc } + } + + /// Transmutes a `ListArc` into an [`Arc`] without updating the tracking inside `T`. + /// + /// After this call, the tracking inside `T` will still think that there is a `ListArc` + /// reference. + #[inline] + fn transmute_to_arc(self) -> Arc { + // Use a transmute to skip destructor. + // + // SAFETY: ListArc is repr(transparent). + unsafe { core::mem::transmute(self) } + } + + /// Convert ownership of this `ListArc` into a raw pointer. + /// + /// The returned pointer is indistinguishable from pointers returned by [`Arc::into_raw`]. The + /// tracking inside `T` will still think that a `ListArc` exists after this call. + #[inline] + pub fn into_raw(self) -> *const T { + Arc::into_raw(Self::transmute_to_arc(self)) + } + + /// Take ownership of the `ListArc` from a raw pointer. + /// + /// # Safety + /// + /// * `ptr` must satisfy the safety requirements of [`Arc::from_raw`]. + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The pointer satisfies the safety requirements for `Arc::from_raw`. + let arc = unsafe { Arc::from_raw(ptr) }; + // SAFETY: The value doesn't already have a `ListArc` reference, but the tracking thinks it + // does. + unsafe { Self::transmute_from_arc(arc) } + } + + /// Converts the `ListArc` into an [`Arc`]. + #[inline] + pub fn into_arc(self) -> Arc { + let arc = Self::transmute_to_arc(self); + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is. + unsafe { T::on_drop_list_arc(&arc) }; + arc + } + + /// Clone a `ListArc` into an [`Arc`]. + #[inline] + pub fn clone_arc(&self) -> Arc { + self.arc.clone() + } + + /// Returns a reference to an [`Arc`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`&Arc`] (e.g., in a method + /// receiver), but we have a [`ListArc`] instead. + /// + /// [`&Arc`]: Arc + #[inline] + pub fn as_arc(&self) -> &Arc { + &self.arc + } + + /// Returns an [`ArcBorrow`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method + /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. + #[inline] + pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { + self.arc.as_arc_borrow() + } + + /// Compare whether two [`ListArc`] pointers reference the same underlying object. + #[inline] + pub fn ptr_eq(this: &Self, other: &Self) -> bool { + Arc::ptr_eq(&this.arc, &other.arc) + } +} + +impl Deref for ListArc +where + T: ListArcSafe + ?Sized, +{ + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.arc.deref() + } +} + +impl Drop for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is by the type + // invariants on `Self`. + unsafe { T::on_drop_list_arc(&self.arc) }; + } +} + +impl AsRef> for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn as_ref(&self) -> &Arc { + self.as_arc() + } +} + +/// A utility for tracking whether a [`ListArc`] exists using an atomic. +/// +/// # Invariants +/// +/// If the boolean is `false`, then there is no [`ListArc`] for this value. +#[repr(transparent)] +pub struct AtomicTracker { + inner: AtomicFlag, + // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. + _pin: PhantomPinned, +} + +impl AtomicTracker { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: AtomicFlag::new(false), + _pin: PhantomPinned, + } + } + + fn project_inner(self: Pin<&mut Self>) -> &mut AtomicFlag { + // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable + // reference to it even if we only have a pinned reference to `self`. + unsafe { &mut Pin::into_inner_unchecked(self).inner } + } +} + +impl ListArcSafe for AtomicTracker { + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>) { + // INVARIANT: We just created a ListArc, so the boolean should be true. + *self.project_inner().get_mut() = true; + } + + unsafe fn on_drop_list_arc(&self) { + // INVARIANT: We just dropped a ListArc, so the boolean should be false. + self.inner.store(false, ordering::Release); + } +} + +// SAFETY: If this method returns `true`, then by the type invariant there is no `ListArc` before +// this call, so it is okay to create a new `ListArc`. +// +// The acquire ordering will synchronize with the release store from the destruction of any +// previous `ListArc`, so if there was a previous `ListArc`, then the destruction of the previous +// `ListArc` happens-before the creation of the new `ListArc`. +unsafe impl TryNewListArc for AtomicTracker { + fn try_new_list_arc(&self) -> bool { + // INVARIANT: If this method returns true, then the boolean used to be false, and is no + // longer false, so it is okay for the caller to create a new [`ListArc`]. + self.inner.cmpxchg(false, true, ordering::Acquire).is_ok() + } +} diff --git a/rust/kernel/list/arc_field.rs b/rust/kernel/list/arc_field.rs new file mode 100644 index 000000000..2ad8aea55 --- /dev/null +++ b/rust/kernel/list/arc_field.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A field that is exclusively owned by a [`ListArc`]. +//! +//! This can be used to have reference counted struct where one of the reference counted pointers +//! has exclusive access to a field of the struct. +//! +//! [`ListArc`]: crate::list::ListArc + +use core::cell::UnsafeCell; + +/// A field owned by a specific [`ListArc`]. +/// +/// [`ListArc`]: crate::list::ListArc +pub struct ListArcField { + value: UnsafeCell, +} + +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Send for ListArcField {} +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Sync for ListArcField {} + +impl ListArcField { + /// Creates a new `ListArcField`. + pub fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + } + } + + /// Access the value when we have exclusive access to the `ListArcField`. + /// + /// This allows access to the field using an `UniqueArc` instead of a `ListArc`. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } + + /// Unsafely assert that you have shared access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have shared access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + pub unsafe fn assert_ref(&self) -> &T { + // SAFETY: The caller has shared access to the `ListArc`, so they also have shared access + // to this field. + unsafe { &*self.value.get() } + } + + /// Unsafely assert that you have mutable access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have mutable access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + #[expect(clippy::mut_from_ref)] + pub unsafe fn assert_mut(&self) -> &mut T { + // SAFETY: The caller has exclusive access to the `ListArc`, so they also have exclusive + // access to this field. + unsafe { &mut *self.value.get() } + } +} + +/// Defines getters for a [`ListArcField`]. +#[macro_export] +#[doc(hidden)] +macro_rules! define_list_arc_field_getter { + ($pub:vis fn $name:ident(&self $(<$id:tt>)?) -> &$typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a $crate::list::ListArc) -> &'a $typ { + let field = &(&**self).$field; + // SAFETY: We have a shared reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_ref(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + ($pub:vis fn $name:ident(&mut self $(<$id:tt>)?) -> &mut $typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a mut $crate::list::ListArc) -> &'a mut $typ { + let field = &(&**self).$field; + // SAFETY: We have a mutable reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_mut(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + () => {}; +} +pub use define_list_arc_field_getter; diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs new file mode 100644 index 000000000..5a3eac9f3 --- /dev/null +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Helpers for implementing list traits safely. + +/// Declares that this type has a [`ListLinks`] field. +/// +/// This trait is only used to help implement [`ListItem`] safely. If [`ListItem`] is implemented +/// manually, then this trait is not needed. Use the [`impl_has_list_links!`] macro to implement +/// this trait. +/// +/// # Safety +/// +/// The methods on this trait must have exactly the behavior that the definitions given below have. +/// +/// [`ListLinks`]: crate::list::ListLinks +/// [`ListItem`]: crate::list::ListItem +pub unsafe trait HasListLinks { + /// Returns a pointer to the [`ListLinks`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + /// + /// [`ListLinks`]: crate::list::ListLinks + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut crate::list::ListLinks; +} + +/// Implements the [`HasListLinks`] trait for the given type. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_has_list_links { + ($(impl$({$($generics:tt)*})? + HasListLinks$(<$id:tt>)? + for $self:ty + { self$(.$field:ident)* } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + // Statically ensure that `$(.field)*` doesn't follow any pointers. + // + // Cannot be `const` because `$self` may contain generics and E0401 says constants + // "can't use {`Self`,generic parameters} from outer item". + if false { let _: usize = ::core::mem::offset_of!(Self, $($field).*); } + + // SAFETY: The caller promises that the pointer is not dangling. We know that this + // expression doesn't follow any pointers, as the `offset_of!` invocation above + // would otherwise not compile. + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) } + } + } + )*}; +} +pub use impl_has_list_links; + +/// Declares that the [`ListLinks`] field in this struct is inside a +/// [`ListLinksSelfPtr`]. +/// +/// # Safety +/// +/// The [`ListLinks`] field of this struct at [`HasListLinks::raw_get_list_links`] must be +/// inside a [`ListLinksSelfPtr`]. +/// +/// [`ListLinks`]: crate::list::ListLinks +/// [`ListLinksSelfPtr`]: crate::list::ListLinksSelfPtr +pub unsafe trait HasSelfPtr +where + Self: HasListLinks, +{ +} + +/// Implements the [`HasListLinks`] and [`HasSelfPtr`] traits for the given type. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_has_list_links_self_ptr { + ($(impl$({$($generics:tt)*})? + HasSelfPtr<$item_type:ty $(, $id:tt)?> + for $self:ty + { self$(.$field:ident)* } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + + // SAFETY: TODO. + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; + ptr.cast() + } + } + )*}; +} +pub use impl_has_list_links_self_ptr; + +/// Implements the [`ListItem`] trait for the given type. +/// +/// Requires that the type implements [`HasListLinks`]. Use the [`impl_has_list_links!`] macro to +/// implement that trait. +/// +/// [`ListItem`]: crate::list::ListItem +/// +/// # Examples +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinks, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinks { self.links }; } +/// } +/// +/// struct ListLinksHolder { +/// inner: kernel::list::ListLinks, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem { +/// value: Result, +/// #[pin] +/// links: ListLinksHolder, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem { using ListLinks { self.links.inner }; } +/// } +/// ``` +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinksSelfPtr, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// struct ListLinksSelfPtrHolder { +/// inner: kernel::list::ListLinksSelfPtr>, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem { +/// value: Result, +/// #[pin] +/// links: ListLinksSelfPtrHolder, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem { +/// using ListLinksSelfPtr { self.links.inner }; +/// } +/// } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! impl_list_item { + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinks { self$(.$field:ident)* }; + })* + ) => {$( + $crate::list::impl_has_list_links! { + impl$({$($generics)*})? HasListLinks<$num> for $self { self$(.$field)* } + } + + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // is implemented in terms of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller guarantees that `me` points at a valid value of type `Self`. + unsafe { + >::raw_get_list_links(me.cast_mut()) + } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + unsafe { $crate::container_of!(me, Self, $($field).*) } + } + + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value. + unsafe { >::view_links(me) } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + unsafe { $crate::container_of!(me, Self, $($field).*) } + } + } + )*}; + + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinksSelfPtr { self$(.$field:ident)* }; + })* + ) => {$( + $crate::list::impl_has_list_links_self_ptr! { + impl$({$($generics)*})? HasSelfPtr<$self> for $self { self$(.$field)* } + } + + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + let links_field = unsafe { >::view_links(me) }; + + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); + + // SAFETY: This value is not accessed in any other places than `prepare_to_insert`, + // `post_remove`, or `view_value`. By the safety requirements of those methods, + // none of these three methods may be called in parallel with this call to + // `prepare_to_insert`, so this write will not race with any other access to the + // value. + unsafe { ::core::ptr::write(cell_inner, me) }; + + links_field + } + + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // returns the return value of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + unsafe { + >::raw_get_list_links(me.cast_mut()) + } + } + + // This function is also used as the implementation of `post_remove`, so the caller + // may choose to satisfy the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + // + // GUARANTEES: (always) + // * This returns the same pointer as the one passed to the most recent call to + // `prepare_to_insert` since that call wrote that pointer to this location. The value + // is only modified in `prepare_to_insert`, so it has not been modified since the + // most recent call. + // + // GUARANTEES: (only when using the `view_value` safety requirements) + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); + + // SAFETY: This is not a data race, because the only function that writes to this + // value is `prepare_to_insert`, but by the safety requirements the + // `prepare_to_insert` method may not be called in parallel with `view_value` or + // `post_remove`. + unsafe { ::core::ptr::read(cell_inner) } + } + + // GUARANTEES: + // The first guarantee of `view_value` is exactly what `post_remove` guarantees. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: This specific implementation of `view_value` allows the caller to + // promise the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + unsafe { >::view_value(me) } + } + } + )*}; +} +pub use impl_list_item; diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs new file mode 100644 index 000000000..265d6396a --- /dev/null +++ b/rust/kernel/maple_tree.rs @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Maple trees. +//! +//! C header: [`include/linux/maple_tree.h`](srctree/include/linux/maple_tree.h) +//! +//! Reference: + +use core::{ + marker::PhantomData, + ops::{Bound, RangeBounds}, + ptr, +}; + +use kernel::{ + alloc::Flags, + error::to_result, + prelude::*, + types::{ForeignOwnable, Opaque}, +}; + +/// A maple tree optimized for storing non-overlapping ranges. +/// +/// # Invariants +/// +/// Each range in the maple tree owns an instance of `T`. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct MapleTree { + #[pin] + tree: Opaque, + _p: PhantomData, +} + +/// A maple tree with `MT_FLAGS_ALLOC_RANGE` set. +/// +/// All methods on [`MapleTree`] are also accessible on this type. +#[pin_data] +#[repr(transparent)] +pub struct MapleTreeAlloc { + #[pin] + tree: MapleTree, +} + +// Make MapleTree methods usable on MapleTreeAlloc. +impl core::ops::Deref for MapleTreeAlloc { + type Target = MapleTree; + + #[inline] + fn deref(&self) -> &MapleTree { + &self.tree + } +} + +#[inline] +fn to_maple_range(range: impl RangeBounds) -> Option<(usize, usize)> { + let first = match range.start_bound() { + Bound::Included(start) => *start, + Bound::Excluded(start) => start.checked_add(1)?, + Bound::Unbounded => 0, + }; + + let last = match range.end_bound() { + Bound::Included(end) => *end, + Bound::Excluded(end) => end.checked_sub(1)?, + Bound::Unbounded => usize::MAX, + }; + + if last < first { + return None; + } + + Some((first, last)) +} + +impl MapleTree { + /// Create a new maple tree. + /// + /// The tree will use the regular implementation with a higher branching factor, rather than + /// the allocation tree. + #[inline] + pub fn new() -> impl PinInit { + pin_init!(MapleTree { + // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be + // destroyed in Drop before the memory location becomes invalid. + tree <- Opaque::ffi_init(|slot| unsafe { bindings::mt_init_flags(slot, 0) }), + _p: PhantomData, + }) + } + + /// Insert the value at the given index. + /// + /// # Errors + /// + /// If the maple tree already contains a range using the given index, then this call will + /// return an [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{InsertErrorKind, MapleTree}; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let the_answer = KBox::new(42, GFP_KERNEL)?; + /// + /// // These calls will succeed. + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(101, twenty, GFP_KERNEL)?; + /// + /// // This will fail because the index is already in use. + /// assert_eq!( + /// tree.insert(100, the_answer, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::Occupied, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn insert(&self, index: usize, value: T, gfp: Flags) -> Result<(), InsertError> { + self.insert_range(index..=index, value, gfp) + } + + /// Insert a value to the specified range, failing on overlap. + /// + /// This accepts the usual types of Rust ranges using the `..` and `..=` syntax for exclusive + /// and inclusive ranges respectively. The range must not be empty, and must not overlap with + /// any existing range. + /// + /// # Errors + /// + /// If the maple tree already contains an overlapping range, then this call will return an + /// [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails or if the + /// requested range is invalid (e.g. empty). + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{InsertErrorKind, MapleTree}; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let the_answer = KBox::new(42, GFP_KERNEL)?; + /// let hundred = KBox::new(100, GFP_KERNEL)?; + /// + /// // Insert the value 10 at the indices 100 to 499. + /// tree.insert_range(100..500, ten, GFP_KERNEL)?; + /// + /// // Insert the value 20 at the indices 500 to 1000. + /// tree.insert_range(500..=1000, twenty, GFP_KERNEL)?; + /// + /// // This will fail due to overlap with the previous range on index 1000. + /// assert_eq!( + /// tree.insert_range(1000..1200, the_answer, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::Occupied, + /// ); + /// + /// // When using .. to specify the range, you must be careful to ensure that the range is + /// // non-empty. + /// assert_eq!( + /// tree.insert_range(72..72, hundred, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::InvalidRequest, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + pub fn insert_range(&self, range: R, value: T, gfp: Flags) -> Result<(), InsertError> + where + R: RangeBounds, + { + let Some((first, last)) = to_maple_range(range) else { + return Err(InsertError { + value, + cause: InsertErrorKind::InvalidRequest, + }); + }; + + let ptr = T::into_foreign(value); + + // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`. + let res = to_result(unsafe { + bindings::mtree_insert_range(self.tree.get(), first, last, ptr, gfp.as_raw()) + }); + + if let Err(err) = res { + // SAFETY: As `mtree_insert_range` failed, it is safe to take back ownership. + let value = unsafe { T::from_foreign(ptr) }; + + let cause = if err == ENOMEM { + InsertErrorKind::AllocError(kernel::alloc::AllocError) + } else if err == EEXIST { + InsertErrorKind::Occupied + } else { + InsertErrorKind::InvalidRequest + }; + Err(InsertError { value, cause }) + } else { + Ok(()) + } + } + + /// Erase the range containing the given index. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// + /// tree.insert_range(100..500, ten, GFP_KERNEL)?; + /// tree.insert(67, twenty, GFP_KERNEL)?; + /// + /// assert_eq!(tree.erase(67).map(|v| *v), Some(20)); + /// assert_eq!(tree.erase(275).map(|v| *v), Some(10)); + /// + /// // The previous call erased the entire range, not just index 275. + /// assert!(tree.erase(127).is_none()); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn erase(&self, index: usize) -> Option { + // SAFETY: `self.tree` contains a valid maple tree. + let ret = unsafe { bindings::mtree_erase(self.tree.get(), index) }; + + // SAFETY: If the pointer is not null, then we took ownership of a valid instance of `T` + // from the tree. + unsafe { T::try_from_foreign(ret) } + } + + /// Lock the internal spinlock. + #[inline] + pub fn lock(&self) -> MapleGuard<'_, T> { + // SAFETY: It's safe to lock the spinlock in a maple tree. + unsafe { bindings::spin_lock(self.ma_lock()) }; + + // INVARIANT: We just took the spinlock. + MapleGuard(self) + } + + #[inline] + fn ma_lock(&self) -> *mut bindings::spinlock_t { + // SAFETY: This pointer offset operation stays in-bounds. + let lock_ptr = unsafe { &raw mut (*self.tree.get()).__bindgen_anon_1.ma_lock }; + lock_ptr.cast() + } + + /// Free all `T` instances in this tree. + /// + /// # Safety + /// + /// This frees Rust data referenced by the maple tree without removing it from the maple tree, + /// leaving it in an invalid state. The caller must ensure that this invalid state cannot be + /// observed by the end-user. + unsafe fn free_all_entries(self: Pin<&mut Self>) { + // SAFETY: The caller provides exclusive access to the entire maple tree, so we have + // exclusive access to the entire maple tree despite not holding the lock. + let mut ma_state = unsafe { MaState::new_raw(self.into_ref().get_ref(), 0, usize::MAX) }; + + loop { + // This uses the raw accessor because we're destroying pointers without removing them + // from the maple tree, which is only valid because this is the destructor. + // + // Take the rcu lock because mas_find_raw() requires that you hold either the spinlock + // or the rcu read lock. This is only really required if memory reclaim might + // reallocate entries in the tree, as we otherwise have exclusive access. That feature + // doesn't exist yet, so for now, taking the rcu lock only serves the purpose of + // silencing lockdep. + let ptr = { + let _rcu = kernel::sync::rcu::Guard::new(); + ma_state.mas_find_raw(usize::MAX) + }; + if ptr.is_null() { + break; + } + // SAFETY: By the type invariants, this pointer references a valid value of type `T`. + // By the safety requirements, it is okay to free it without removing it from the maple + // tree. + drop(unsafe { T::from_foreign(ptr) }); + } + } +} + +#[pinned_drop] +impl PinnedDrop for MapleTree { + #[inline] + fn drop(mut self: Pin<&mut Self>) { + // We only iterate the tree if the Rust value has a destructor. + if core::mem::needs_drop::() { + // SAFETY: Other than the below `mtree_destroy` call, the tree will not be accessed + // after this call. + unsafe { self.as_mut().free_all_entries() }; + } + + // SAFETY: The tree is valid, and will not be accessed after this call. + unsafe { bindings::mtree_destroy(self.tree.get()) }; + } +} + +/// A reference to a [`MapleTree`] that owns the inner lock. +/// +/// # Invariants +/// +/// This guard owns the inner spinlock. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree); + +impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, we hold this spinlock. + unsafe { bindings::spin_unlock(self.0.ma_lock()) }; + } +} + +impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { + /// Create a [`MaState`] protected by this lock guard. + pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { + // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s + // read/write permissions to the maple tree. + unsafe { MaState::new_raw(self.0, first, end) } + } + + /// Load the value at the given index. + /// + /// # Examples + /// + /// Read the value while holding the spinlock. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// let mut lock = tree.lock(); + /// assert_eq!(lock.load(100).map(|v| *v), Some(10)); + /// assert_eq!(lock.load(200).map(|v| *v), Some(20)); + /// assert_eq!(lock.load(300).map(|v| *v), None); + /// # Ok::<_, Error>(()) + /// ``` + /// + /// Increment refcount under the lock, to keep value alive afterwards. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// use kernel::sync::Arc; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = Arc::new(10, GFP_KERNEL)?; + /// let twenty = Arc::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// // Briefly take the lock to increment the refcount. + /// let value = tree.lock().load(100).map(Arc::from); + /// + /// // At this point, another thread might remove the value. + /// tree.erase(100); + /// + /// // But we can still access it because we took a refcount. + /// assert_eq!(value.map(|v| *v), Some(10)); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn load(&mut self, index: usize) -> Option> { + // SAFETY: `self.tree` contains a valid maple tree. + let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) }; + if ret.is_null() { + return None; + } + + // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It is + // safe to borrow the instance mutably because the signature of this function enforces that + // the mutable borrow is not used after the spinlock is dropped. + Some(unsafe { T::borrow_mut(ret) }) + } +} + +impl MapleTreeAlloc { + /// Create a new allocation tree. + pub fn new() -> impl PinInit { + let tree = pin_init!(MapleTree { + // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be + // destroyed in Drop before the memory location becomes invalid. + tree <- Opaque::ffi_init(|slot| unsafe { + bindings::mt_init_flags(slot, bindings::MT_FLAGS_ALLOC_RANGE) + }), + _p: PhantomData, + }); + + pin_init!(MapleTreeAlloc { tree <- tree }) + } + + /// Insert an entry with the given size somewhere in the given range. + /// + /// The maple tree will search for a location in the given range where there is space to insert + /// the new range. If there is not enough available space, then an error will be returned. + /// + /// The index of the new range is returned. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{MapleTreeAlloc, AllocErrorKind}; + /// + /// let tree = KBox::pin_init(MapleTreeAlloc::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let thirty = KBox::new(30, GFP_KERNEL)?; + /// let hundred = KBox::new(100, GFP_KERNEL)?; + /// + /// // Allocate three ranges. + /// let idx1 = tree.alloc_range(100, ten, ..1000, GFP_KERNEL)?; + /// let idx2 = tree.alloc_range(100, twenty, ..1000, GFP_KERNEL)?; + /// let idx3 = tree.alloc_range(100, thirty, ..1000, GFP_KERNEL)?; + /// + /// assert_eq!(idx1, 0); + /// assert_eq!(idx2, 100); + /// assert_eq!(idx3, 200); + /// + /// // This will fail because the remaining space is too small. + /// assert_eq!( + /// tree.alloc_range(800, hundred, ..1000, GFP_KERNEL).unwrap_err().cause, + /// AllocErrorKind::Busy, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + pub fn alloc_range( + &self, + size: usize, + value: T, + range: R, + gfp: Flags, + ) -> Result> + where + R: RangeBounds, + { + let Some((min, max)) = to_maple_range(range) else { + return Err(AllocError { + value, + cause: AllocErrorKind::InvalidRequest, + }); + }; + + let ptr = T::into_foreign(value); + let mut index = 0; + + // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`. + let res = to_result(unsafe { + bindings::mtree_alloc_range( + self.tree.tree.get(), + &mut index, + ptr, + size, + min, + max, + gfp.as_raw(), + ) + }); + + if let Err(err) = res { + // SAFETY: As `mtree_alloc_range` failed, it is safe to take back ownership. + let value = unsafe { T::from_foreign(ptr) }; + + let cause = if err == ENOMEM { + AllocErrorKind::AllocError(kernel::alloc::AllocError) + } else if err == EBUSY { + AllocErrorKind::Busy + } else { + AllocErrorKind::InvalidRequest + }; + Err(AllocError { value, cause }) + } else { + Ok(index) + } + } +} + +/// A helper type used for navigating a [`MapleTree`]. +/// +/// # Invariants +/// +/// For the duration of `'tree`: +/// +/// * The `ma_state` references a valid `MapleTree`. +/// * The `ma_state` has read/write access to the tree. +pub struct MaState<'tree, T: ForeignOwnable> { + state: bindings::ma_state, + _phantom: PhantomData<&'tree mut MapleTree>, +} + +impl<'tree, T: ForeignOwnable> MaState<'tree, T> { + /// Initialize a new `MaState` with the given tree. + /// + /// # Safety + /// + /// The caller must ensure that this `MaState` has read/write access to the maple tree. + #[inline] + unsafe fn new_raw(mt: &'tree MapleTree, first: usize, end: usize) -> Self { + // INVARIANT: + // * Having a reference ensures that the `MapleTree` is valid for `'tree`. + // * The caller ensures that we have read/write access. + Self { + state: bindings::ma_state { + tree: mt.tree.get(), + index: first, + last: end, + node: ptr::null_mut(), + status: bindings::maple_status_ma_start, + min: 0, + max: usize::MAX, + alloc: ptr::null_mut(), + mas_flags: 0, + store_type: bindings::store_type_wr_invalid, + ..Default::default() + }, + _phantom: PhantomData, + } + } + + #[inline] + fn as_raw(&mut self) -> *mut bindings::ma_state { + &raw mut self.state + } + + #[inline] + fn mas_find_raw(&mut self, max: usize) -> *mut c_void { + // SAFETY: By the type invariants, the `ma_state` is active and we have read/write access + // to the tree. + unsafe { bindings::mas_find(self.as_raw(), max) } + } + + /// Find the next entry in the maple tree. + /// + /// # Examples + /// + /// Iterate the maple tree. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// use kernel::sync::Arc; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = Arc::new(10, GFP_KERNEL)?; + /// let twenty = Arc::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// let mut ma_lock = tree.lock(); + /// let mut iter = ma_lock.ma_state(0, usize::MAX); + /// + /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(10)); + /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(20)); + /// assert!(iter.find(usize::MAX).is_none()); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn find(&mut self, max: usize) -> Option> { + let ret = self.mas_find_raw(max); + if ret.is_null() { + return None; + } + + // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It's + // safe to access it mutably as the returned reference borrows this `MaState`, and the + // `MaState` has read/write access to the maple tree. + Some(unsafe { T::borrow_mut(ret) }) + } +} + +/// Error type for failure to insert a new value. +pub struct InsertError { + /// The value that could not be inserted. + pub value: T, + /// The reason for the failure to insert. + pub cause: InsertErrorKind, +} + +/// The reason for the failure to insert. +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum InsertErrorKind { + /// There is already a value in the requested range. + Occupied, + /// Failure to allocate memory. + AllocError(kernel::alloc::AllocError), + /// The insertion request was invalid. + InvalidRequest, +} + +impl From for Error { + #[inline] + fn from(kind: InsertErrorKind) -> Error { + match kind { + InsertErrorKind::Occupied => EEXIST, + InsertErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM, + InsertErrorKind::InvalidRequest => EINVAL, + } + } +} + +impl From> for Error { + #[inline] + fn from(insert_err: InsertError) -> Error { + Error::from(insert_err.cause) + } +} + +/// Error type for failure to insert a new value. +pub struct AllocError { + /// The value that could not be inserted. + pub value: T, + /// The reason for the failure to insert. + pub cause: AllocErrorKind, +} + +/// The reason for the failure to insert. +#[derive(PartialEq, Eq, Copy, Clone)] +pub enum AllocErrorKind { + /// There is not enough space for the requested allocation. + Busy, + /// Failure to allocate memory. + AllocError(kernel::alloc::AllocError), + /// The insertion request was invalid. + InvalidRequest, +} + +impl From for Error { + #[inline] + fn from(kind: AllocErrorKind) -> Error { + match kind { + AllocErrorKind::Busy => EBUSY, + AllocErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM, + AllocErrorKind::InvalidRequest => EINVAL, + } + } +} + +impl From> for Error { + #[inline] + fn from(insert_err: AllocError) -> Error { + Error::from(insert_err.cause) + } +} diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs new file mode 100644 index 000000000..8d4b719bd --- /dev/null +++ b/rust/kernel/miscdevice.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Miscdevice support. +//! +//! C headers: [`include/linux/miscdevice.h`](srctree/include/linux/miscdevice.h). +//! +//! Reference: + +use crate::{ + bindings, + device::Device, + error::{ + to_result, + VTABLE_DEFAULT_ERROR, // + }, + fs::{ + File, + Kiocb, // + }, + iov::{ + IovIterDest, + IovIterSource, // + }, + mm::virt::VmaNew, + module::this_module, + prelude::*, + seq_file::SeqFile, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::marker::PhantomData; + +/// Options for creating a misc device. +#[derive(Copy, Clone)] +pub struct MiscDeviceOptions { + /// The name of the miscdevice. + pub name: &'static CStr, +} + +impl MiscDeviceOptions { + /// Create a raw `struct miscdev` ready for registration. + pub const fn into_raw(self) -> bindings::miscdevice { + let mut result: bindings::miscdevice = pin_init::zeroed(); + result.minor = bindings::MISC_DYNAMIC_MINOR as ffi::c_int; + result.name = crate::str::as_char_ptr_in_const_context(self.name); + result.fops = MiscdeviceVTable::::build(); + result + } +} + +/// A registration of a miscdevice. +/// +/// # Invariants +/// +/// - `inner` contains a `struct miscdevice` that is registered using +/// `misc_register()`. +/// - This registration remains valid for the entire lifetime of the +/// [`MiscDeviceRegistration`] instance. +/// - Deregistration occurs exactly once in [`Drop`] via `misc_deregister()`. +/// - `inner` wraps a valid, pinned `miscdevice` created using +/// [`MiscDeviceOptions::into_raw`]. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct MiscDeviceRegistration { + #[pin] + inner: Opaque, + _t: PhantomData, +} + +// SAFETY: It is allowed to call `misc_deregister` on a different thread from where you called +// `misc_register`. +unsafe impl Send for MiscDeviceRegistration {} +// SAFETY: All `&self` methods on this type are written to ensure that it is safe to call them in +// parallel. +unsafe impl Sync for MiscDeviceRegistration {} + +impl MiscDeviceRegistration { + /// Register a misc device. + pub fn register(opts: MiscDeviceOptions) -> impl PinInit { + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(move |slot: *mut bindings::miscdevice| { + // SAFETY: The initializer can write to the provided `slot`. + unsafe { slot.write(opts.into_raw::()) }; + + // SAFETY: We just wrote the misc device options to the slot. The miscdevice will + // get unregistered before `slot` is deallocated because the memory is pinned and + // the destructor of this type deallocates the memory. + // INVARIANT: If this returns `Ok(())`, then the `slot` will contain a registered + // misc device. + to_result(unsafe { bindings::misc_register(slot) }) + }), + _t: PhantomData, + }) + } + + /// Returns a raw pointer to the misc device. + pub fn as_raw(&self) -> *mut bindings::miscdevice { + self.inner.get() + } + + /// Access the `this_device` field. + pub fn device(&self) -> &Device { + // SAFETY: This can only be called after a successful register(), which always + // initialises `this_device` with a valid device. Furthermore, the signature of this + // function tells the borrow-checker that the `&Device` reference must not outlive the + // `&MiscDeviceRegistration` used to obtain it, so the last use of the reference must be + // before the underlying `struct miscdevice` is destroyed. + unsafe { Device::from_raw((*self.as_raw()).this_device) } + } +} + +#[pinned_drop] +impl PinnedDrop for MiscDeviceRegistration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: We know that the device is registered by the type invariants. + unsafe { bindings::misc_deregister(self.inner.get()) }; + } +} + +/// Trait implemented by the private data of an open misc device. +#[vtable] +pub trait MiscDevice: Sized { + /// What kind of pointer should `Self` be wrapped in. + type Ptr: ForeignOwnable + Send + Sync; + + /// Called when the misc device is opened. + /// + /// The returned pointer will be stored as the private data for the file. + fn open(_file: &File, _misc: &MiscDeviceRegistration) -> Result; + + /// Called when the misc device is released. + fn release(device: Self::Ptr, _file: &File) { + drop(device); + } + + /// Handle for mmap. + /// + /// This function is invoked when a user space process invokes the `mmap` system call on + /// `file`. The function is a callback that is part of the VMA initializer. The kernel will do + /// initial setup of the VMA before calling this function. The function can then interact with + /// the VMA initialization by calling methods of `vma`. If the function does not return an + /// error, the kernel will complete initialization of the VMA according to the properties of + /// `vma`. + fn mmap( + _device: ::Borrowed<'_>, + _file: &File, + _vma: &VmaNew, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Read from this miscdevice. + fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Write to this miscdevice. + fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Handler for ioctls. + /// + /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`]. + /// + /// [`kernel::ioctl`]: mod@crate::ioctl + fn ioctl( + _device: ::Borrowed<'_>, + _file: &File, + _cmd: u32, + _arg: usize, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Handler for ioctls. + /// + /// Used for 32-bit userspace on 64-bit platforms. + /// + /// This method is optional and only needs to be provided if the ioctl relies on structures + /// that have different layout on 32-bit and 64-bit userspace. If no implementation is + /// provided, then `compat_ptr_ioctl` will be used instead. + #[cfg(CONFIG_COMPAT)] + fn compat_ioctl( + _device: ::Borrowed<'_>, + _file: &File, + _cmd: u32, + _arg: usize, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Show info for this fd. + fn show_fdinfo( + _device: ::Borrowed<'_>, + _m: &SeqFile, + _file: &File, + ) { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// A vtable for the file operations of a Rust miscdevice. +struct MiscdeviceVTable(PhantomData); + +impl MiscdeviceVTable { + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is undergoing initialization. + /// The file must be associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn open(inode: *mut bindings::inode, raw_file: *mut bindings::file) -> c_int { + // SAFETY: The pointers are valid and for a file being opened. + let ret = unsafe { bindings::generic_file_open(inode, raw_file) }; + if ret != 0 { + return ret; + } + + // SAFETY: The open call of a file can access the private data. + let misc_ptr = unsafe { (*raw_file).private_data }; + + // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the + // associated `struct miscdevice` before calling into this method. Furthermore, + // `misc_open()` ensures that the miscdevice can't be unregistered and freed during this + // call to `fops_open`. + let misc = unsafe { &*misc_ptr.cast::>() }; + + // SAFETY: + // * This underlying file is valid for (much longer than) the duration of `T::open`. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(raw_file) }; + + let ptr = match T::open(file, misc) { + Ok(ptr) => ptr, + Err(err) => return err.to_errno(), + }; + + // This overwrites the private data with the value specified by the user, changing the type + // of this file's private data. All future accesses to the private data is performed by + // other fops_* methods in this file, which all correctly cast the private data to the new + // type. + // + // SAFETY: The open call of a file can access the private data. + unsafe { (*raw_file).private_data = ptr.into_foreign() }; + + 0 + } + + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is being released. The file + /// must be associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn release(_inode: *mut bindings::inode, file: *mut bindings::file) -> c_int { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: The release call of a file owns the private data. + let ptr = unsafe { ::from_foreign(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + T::release(ptr, unsafe { File::from_raw_file(file) }); + + 0 + } + + /// # Safety + /// + /// `kiocb` must be correspond to a valid file that is associated with a + /// `MiscDeviceRegistration`. `iter` must be a valid `struct iov_iter` for writing. + unsafe extern "C" fn read_iter( + kiocb: *mut bindings::kiocb, + iter: *mut bindings::iov_iter, + ) -> isize { + // SAFETY: The caller provides a valid `struct kiocb` associated with a + // `MiscDeviceRegistration` file. + let kiocb = unsafe { Kiocb::from_raw(kiocb) }; + // SAFETY: This is a valid `struct iov_iter` for writing. + let iov = unsafe { IovIterDest::from_raw(iter) }; + + match T::read_iter(kiocb, iov) { + Ok(res) => res as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `kiocb` must be correspond to a valid file that is associated with a + /// `MiscDeviceRegistration`. `iter` must be a valid `struct iov_iter` for reading. + unsafe extern "C" fn write_iter( + kiocb: *mut bindings::kiocb, + iter: *mut bindings::iov_iter, + ) -> isize { + // SAFETY: The caller provides a valid `struct kiocb` associated with a + // `MiscDeviceRegistration` file. + let kiocb = unsafe { Kiocb::from_raw(kiocb) }; + // SAFETY: This is a valid `struct iov_iter` for reading. + let iov = unsafe { IovIterSource::from_raw(iter) }; + + match T::write_iter(kiocb, iov) { + Ok(res) => res as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + /// `vma` must be a vma that is currently being mmap'ed with this file. + unsafe extern "C" fn mmap( + file: *mut bindings::file, + vma: *mut bindings::vm_area_struct, + ) -> c_int { + // SAFETY: The mmap call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: This is a Rust Miscdevice, so we call `into_foreign` in `open` and + // `from_foreign` in `release`, and `fops_mmap` is guaranteed to be called between those + // two operations. + let device = unsafe { ::borrow(private.cast()) }; + // SAFETY: The caller provides a vma that is undergoing initial VMA setup. + let area = unsafe { VmaNew::from_raw(vma) }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::mmap(device, file, area) { + Ok(()) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn ioctl(file: *mut bindings::file, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: The ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + #[cfg(CONFIG_COMPAT)] + unsafe extern "C" fn compat_ioctl( + file: *mut bindings::file, + cmd: c_uint, + arg: c_ulong, + ) -> c_long { + // SAFETY: The compat ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::compat_ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } + } + + /// # Safety + /// + /// - `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + /// - `seq_file` must be a valid `struct seq_file` that we can write to. + unsafe extern "C" fn show_fdinfo(seq_file: *mut bindings::seq_file, file: *mut bindings::file) { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in + // which this method is called. + let m = unsafe { SeqFile::from_raw(seq_file) }; + + T::show_fdinfo(device, m, file); + } + + const VTABLE: bindings::file_operations = bindings::file_operations { + open: Some(Self::open), + release: Some(Self::release), + mmap: if T::HAS_MMAP { Some(Self::mmap) } else { None }, + read_iter: if T::HAS_READ_ITER { + Some(Self::read_iter) + } else { + None + }, + write_iter: if T::HAS_WRITE_ITER { + Some(Self::write_iter) + } else { + None + }, + unlocked_ioctl: if T::HAS_IOCTL { + Some(Self::ioctl) + } else { + None + }, + #[cfg(CONFIG_COMPAT)] + compat_ioctl: if T::HAS_COMPAT_IOCTL { + Some(Self::compat_ioctl) + } else if T::HAS_IOCTL { + bindings::compat_ptr_ioctl + } else { + None + }, + show_fdinfo: if T::HAS_SHOW_FDINFO { + Some(Self::show_fdinfo) + } else { + None + }, + owner: this_module::().as_ptr(), + ..pin_init::zeroed() + }; + + const fn build() -> &'static bindings::file_operations { + &Self::VTABLE + } +} diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs new file mode 100644 index 000000000..4764d7b68 --- /dev/null +++ b/rust/kernel/mm.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Memory management. +//! +//! This module deals with managing the address space of userspace processes. Each process has an +//! instance of [`Mm`], which keeps track of multiple VMAs (virtual memory areas). Each VMA +//! corresponds to a region of memory that the userspace process can access, and the VMA lets you +//! control what happens when userspace reads or writes to that region of memory. +//! +//! C header: [`include/linux/mm.h`](srctree/include/linux/mm.h) + +use crate::{ + bindings, + sync::aref::{ARef, AlwaysRefCounted}, + types::{NotThreadSafe, Opaque}, +}; +use core::{ops::Deref, ptr::NonNull}; + +pub mod virt; +use virt::VmaRef; + +#[cfg(CONFIG_MMU)] +pub use mmput_async::MmWithUserAsync; +mod mmput_async; + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This represents the address space of a userspace process, so each process has one `Mm` +/// instance. It may hold many VMAs internally. +/// +/// There is a counter called `mm_users` that counts the users of the address space; this includes +/// the userspace process itself, but can also include kernel threads accessing the address space. +/// Once `mm_users` reaches zero, this indicates that the address space can be destroyed. To access +/// the address space, you must prevent `mm_users` from reaching zero while you are accessing it. +/// The [`MmWithUser`] type represents an address space where this is guaranteed, and you can +/// create one using [`mmget_not_zero`]. +/// +/// The `ARef` smart pointer holds an `mmgrab` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmgrab`. +/// +/// [`mmget_not_zero`]: Mm::mmget_not_zero +#[repr(transparent)] +pub struct Mm { + mm: Opaque, +} + +// SAFETY: It is safe to call `mmdrop` on another thread than where `mmgrab` was called. +unsafe impl Send for Mm {} +// SAFETY: All methods on `Mm` can be called in parallel from several threads. +unsafe impl Sync for Mm {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for Mm { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmgrab(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmdrop(obj.cast().as_ptr()) }; + } +} + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This type is like [`Mm`], but with non-zero `mm_users`. It can only be used when `mm_users` can +/// be proven to be non-zero at compile-time, usually because the relevant code holds an `mmget` +/// refcount. It can be used to access the associated address space. +/// +/// The `ARef` smart pointer holds an `mmget` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero. +#[repr(transparent)] +pub struct MmWithUser { + mm: Mm, +} + +// SAFETY: It is safe to call `mmput` on another thread than where `mmget` was called. +unsafe impl Send for MmWithUser {} +// SAFETY: All methods on `MmWithUser` can be called in parallel from several threads. +unsafe impl Sync for MmWithUser {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for MmWithUser { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmget(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmput(obj.cast().as_ptr()) }; + } +} + +// Make all `Mm` methods available on `MmWithUser`. +impl Deref for MmWithUser { + type Target = Mm; + + #[inline] + fn deref(&self) -> &Mm { + &self.mm + } +} + +// These methods are safe to call even if `mm_users` is zero. +impl Mm { + /// Returns a raw pointer to the inner `mm_struct`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::mm_struct { + self.mm.get() + } + + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that it is not deallocated + /// during the lifetime 'a. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a Mm { + // SAFETY: Caller promises that the pointer is valid for 'a. Layouts are compatible due to + // repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Calls `mmget_not_zero` and returns a handle if it succeeds. + #[inline] + pub fn mmget_not_zero(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmget_not_zero(self.as_raw()) }; + + if success { + // SAFETY: We just created an `mmget` refcount. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(self.as_raw().cast())) }) + } else { + None + } + } +} + +// These methods require `mm_users` to be non-zero. +impl MmWithUser { + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that `mm_users` remains + /// non-zero for the duration of the lifetime 'a. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser { + // SAFETY: Caller promises that the pointer is valid for 'a. The layout is compatible due + // to repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Attempt to access a vma using the vma read lock. + /// + /// This is an optimistic trylock operation, so it may fail if there is contention. In that + /// case, you should fall back to taking the mmap read lock. + /// + /// When per-vma locks are disabled, this always returns `None`. + #[inline] + pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { + #[cfg(CONFIG_PER_VMA_LOCK)] + { + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if !vma.is_null() { + return Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }); + } + } + + // Silence warnings about unused variables. + #[cfg(not(CONFIG_PER_VMA_LOCK))] + let _ = vma_addr; + + None + } + + /// Lock the mmap read lock. + #[inline] + pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmap_read_lock(self.as_raw()) }; + + // INVARIANT: We just acquired the read lock. + MmapReadGuard { + mm: self, + _nts: NotThreadSafe, + } + } + + /// Try to lock the mmap read lock. + #[inline] + pub fn mmap_read_trylock(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmap_read_trylock(self.as_raw()) }; + + if success { + // INVARIANT: We just acquired the read lock. + Some(MmapReadGuard { + mm: self, + _nts: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard for the mmap read lock. +/// +/// # Invariants +/// +/// This `MmapReadGuard` guard owns the mmap read lock. +pub struct MmapReadGuard<'a> { + mm: &'a MmWithUser, + // `mmap_read_lock` and `mmap_read_unlock` must be called on the same thread + _nts: NotThreadSafe, +} + +impl<'a> MmapReadGuard<'a> { + /// Look up a vma at the given address. + #[inline] + pub fn vma_lookup(&self, vma_addr: usize) -> Option<&virt::VmaRef> { + // SAFETY: By the type invariants we hold the mmap read guard, so we can safely call this + // method. Any value is okay for `vma_addr`. + let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr) }; + + if vma.is_null() { + None + } else { + // SAFETY: We just checked that a vma was found, so the pointer references a valid vma. + // + // Furthermore, the returned vma is still under the protection of the read lock guard + // and can be used while the mmap read lock is still held. That the vma is not used + // after the MmapReadGuard gets dropped is enforced by the borrow-checker. + unsafe { Some(virt::VmaRef::from_raw(vma)) } + } + } +} + +impl Drop for MmapReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) }; + } +} + +/// A guard for the vma read lock. +/// +/// # Invariants +/// +/// This `VmaReadGuard` guard owns the vma read lock. +pub struct VmaReadGuard<'a> { + vma: &'a VmaRef, + // `vma_end_read` must be called on the same thread as where the lock was taken + _nts: NotThreadSafe, +} + +// Make all `VmaRef` methods available on `VmaReadGuard`. +impl Deref for VmaReadGuard<'_> { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + self.vma + } +} + +impl Drop for VmaReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::vma_end_read(self.vma.as_ptr()) }; + } +} diff --git a/rust/kernel/mm/mmput_async.rs b/rust/kernel/mm/mmput_async.rs new file mode 100644 index 000000000..b8d2f0512 --- /dev/null +++ b/rust/kernel/mm/mmput_async.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Version of `MmWithUser` using `mmput_async`. +//! +//! This is a separate file from `mm.rs` due to the dependency on `CONFIG_MMU=y`. +#![cfg(CONFIG_MMU)] + +use crate::{ + bindings, + mm::MmWithUser, + sync::aref::{ARef, AlwaysRefCounted}, +}; +use core::{ops::Deref, ptr::NonNull}; + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This type is identical to `MmWithUser` except that it uses `mmput_async` when dropping a +/// refcount. This means that the destructor of `ARef` is safe to call in atomic +/// context. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero. +#[repr(transparent)] +pub struct MmWithUserAsync { + mm: MmWithUser, +} + +// SAFETY: It is safe to call `mmput_async` on another thread than where `mmget` was called. +unsafe impl Send for MmWithUserAsync {} +// SAFETY: All methods on `MmWithUserAsync` can be called in parallel from several threads. +unsafe impl Sync for MmWithUserAsync {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for MmWithUserAsync { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmget(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmput_async(obj.cast().as_ptr()) }; + } +} + +// Make all `MmWithUser` methods available on `MmWithUserAsync`. +impl Deref for MmWithUserAsync { + type Target = MmWithUser; + + #[inline] + fn deref(&self) -> &MmWithUser { + &self.mm + } +} + +impl MmWithUser { + /// Use `mmput_async` when dropping this refcount. + #[inline] + pub fn into_mmput_async(me: ARef) -> ARef { + // SAFETY: The layouts and invariants are compatible. + unsafe { ARef::from_raw(ARef::into_raw(me).cast()) } + } +} diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs new file mode 100644 index 000000000..63eb730b0 --- /dev/null +++ b/rust/kernel/mm/virt.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Virtual memory. +//! +//! This module deals with managing a single VMA in the address space of a userspace process. Each +//! VMA corresponds to a region of memory that the userspace process can access, and the VMA lets +//! you control what happens when userspace reads or writes to that region of memory. +//! +//! The module has several different Rust types that all correspond to the C type called +//! `vm_area_struct`. The different structs represent what kind of access you have to the VMA, e.g. +//! [`VmaRef`] is used when you hold the mmap or vma read lock. Using the appropriate struct +//! ensures that you can't, for example, accidentally call a function that requires holding the +//! write lock when you only hold the read lock. + +use crate::{ + bindings, + error::{code::EINVAL, to_result, Result}, + mm::MmWithUser, + page::Page, + types::Opaque, +}; + +use core::ops::Deref; + +/// A wrapper for the kernel's `struct vm_area_struct` with read access. +/// +/// It represents an area of virtual memory. +/// +/// # Invariants +/// +/// The caller must hold the mmap read lock or the vma read lock. +#[repr(transparent)] +pub struct VmaRef { + vma: Opaque, +} + +// Methods you can call when holding the mmap or vma read lock (or stronger). They must be usable +// no matter what the vma flags are. +impl VmaRef { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap or vma + /// read lock (or stronger) is held for at least the duration of 'a. + #[inline] + pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Returns a raw pointer to this area. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::vm_area_struct { + self.vma.get() + } + + /// Access the underlying `mm_struct`. + #[inline] + pub fn mm(&self) -> &MmWithUser { + // SAFETY: By the type invariants, this `vm_area_struct` is valid and we hold the mmap/vma + // read lock or stronger. This implies that the underlying mm has a non-zero value of + // `mm_users`. + unsafe { MmWithUser::from_raw((*self.as_ptr()).vm_mm) } + } + + /// Returns the flags associated with the virtual memory area. + /// + /// The possible flags are a combination of the constants in [`flags`]. + #[inline] + pub fn flags(&self) -> vm_flags_t { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags } + } + + /// Returns the (inclusive) start address of the virtual memory area. + #[inline] + pub fn start(&self) -> usize { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_start } + } + + /// Returns the (exclusive) end address of the virtual memory area. + #[inline] + pub fn end(&self) -> usize { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_end } + } + + /// Zap pages in the given page range. + /// + /// This clears page table mappings for the range at the leaf level, leaving all other page + /// tables intact, and freeing any memory referenced by the VMA in this range. That is, + /// anonymous memory is completely freed, file-backed memory has its reference count on page + /// cache folio's dropped, any dirty data will still be written back to disk as usual. + /// + /// It may seem odd that we clear at the leaf level, this is however a product of the page + /// table structure used to map physical memory into a virtual address space - each virtual + /// address actually consists of a bitmap of array indices into page tables, which form a + /// hierarchical page table level structure. + /// + /// As a result, each page table level maps a multiple of page table levels below, and thus + /// span ever increasing ranges of pages. At the leaf or PTE level, we map the actual physical + /// memory. + /// + /// It is here where a zap operates, as it the only place we can be certain of clearing without + /// impacting any other virtual mappings. It is an implementation detail as to whether the + /// kernel goes further in freeing unused page tables, but for the purposes of this operation + /// we must only assume that the leaf level is cleared. + #[inline] + pub fn zap_vma_range(&self, address: usize, size: usize) { + let (end, did_overflow) = address.overflowing_add(size); + if did_overflow || address < self.start() || self.end() < end { + // TODO: call WARN_ONCE once Rust version of it is added + return; + } + + // SAFETY: By the type invariants, the caller has read access to this VMA, which is + // sufficient for this method call. This method has no requirements on the vma flags. The + // address range is checked to be within the vma. + unsafe { bindings::zap_vma_range(self.as_ptr(), address, size) }; + } + + /// If the [`VM_MIXEDMAP`] flag is set, returns a [`VmaMixedMap`] to this VMA, otherwise + /// returns `None`. + /// + /// This can be used to access methods that require [`VM_MIXEDMAP`] to be set. + /// + /// [`VM_MIXEDMAP`]: flags::MIXEDMAP + #[inline] + pub fn as_mixedmap_vma(&self) -> Option<&VmaMixedMap> { + if self.flags() & flags::MIXEDMAP != 0 { + // SAFETY: We just checked that `VM_MIXEDMAP` is set. All other requirements are + // satisfied by the type invariants of `VmaRef`. + Some(unsafe { VmaMixedMap::from_raw(self.as_ptr()) }) + } else { + None + } + } +} + +/// A wrapper for the kernel's `struct vm_area_struct` with read access and [`VM_MIXEDMAP`] set. +/// +/// It represents an area of virtual memory. +/// +/// This struct is identical to [`VmaRef`] except that it must only be used when the +/// [`VM_MIXEDMAP`] flag is set on the vma. +/// +/// # Invariants +/// +/// The caller must hold the mmap read lock or the vma read lock. The `VM_MIXEDMAP` flag must be +/// set. +/// +/// [`VM_MIXEDMAP`]: flags::MIXEDMAP +#[repr(transparent)] +pub struct VmaMixedMap { + vma: VmaRef, +} + +// Make all `VmaRef` methods available on `VmaMixedMap`. +impl Deref for VmaMixedMap { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + &self.vma + } +} + +impl VmaMixedMap { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap read lock + /// (or stronger) is held for at least the duration of 'a. The `VM_MIXEDMAP` flag must be set. + #[inline] + pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Maps a single page at the given address within the virtual memory area. + /// + /// This operation does not take ownership of the page. + #[inline] + pub fn vm_insert_page(&self, address: usize, page: &Page) -> Result { + // SAFETY: By the type invariant of `Self` caller has read access and has verified that + // `VM_MIXEDMAP` is set. By invariant on `Page` the page has order 0. + to_result(unsafe { bindings::vm_insert_page(self.as_ptr(), address, page.as_ptr()) }) + } +} + +/// A configuration object for setting up a VMA in an `f_ops->mmap()` hook. +/// +/// The `f_ops->mmap()` hook is called when a new VMA is being created, and the hook is able to +/// configure the VMA in various ways to fit the driver that owns it. Using `VmaNew` indicates that +/// you are allowed to perform operations on the VMA that can only be performed before the VMA is +/// fully initialized. +/// +/// # Invariants +/// +/// For the duration of 'a, the referenced vma must be undergoing initialization in an +/// `f_ops->mmap()` hook. +#[repr(transparent)] +pub struct VmaNew { + vma: VmaRef, +} + +// Make all `VmaRef` methods available on `VmaNew`. +impl Deref for VmaNew { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + &self.vma + } +} + +impl VmaNew { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is undergoing initial vma setup for the duration of 'a. + #[inline] + pub unsafe fn from_raw<'a>(vma: *mut bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Internal method for updating the vma flags. + /// + /// # Safety + /// + /// This must not be used to set the flags to an invalid value. + #[inline] + unsafe fn update_flags(&self, set: vm_flags_t, unset: vm_flags_t) { + let mut flags = self.flags(); + flags |= set; + flags &= !unset; + + // SAFETY: This is not a data race: the vma is undergoing initial setup, so it's not yet + // shared. Additionally, `VmaNew` is `!Sync`, so it cannot be used to write in parallel. + // The caller promises that this does not set the flags to an invalid value. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags = flags }; + } + + /// Set the `VM_MIXEDMAP` flag on this vma. + /// + /// This enables the vma to contain both `struct page` and pure PFN pages. Returns a reference + /// that can be used to call `vm_insert_page` on the vma. + #[inline] + pub fn set_mixedmap(&self) -> &VmaMixedMap { + // SAFETY: We don't yet provide a way to set VM_PFNMAP, so this cannot put the flags in an + // invalid state. + unsafe { self.update_flags(flags::MIXEDMAP, 0) }; + + // SAFETY: We just set `VM_MIXEDMAP` on the vma. + unsafe { VmaMixedMap::from_raw(self.vma.as_ptr()) } + } + + /// Set the `VM_IO` flag on this vma. + /// + /// This is used for memory mapped IO and similar. The flag tells other parts of the kernel to + /// avoid looking at the pages. For memory mapped IO this is useful as accesses to the pages + /// could have side effects. + #[inline] + pub fn set_io(&self) { + // SAFETY: Setting the VM_IO flag is always okay. + unsafe { self.update_flags(flags::IO, 0) }; + } + + /// Set the `VM_DONTEXPAND` flag on this vma. + /// + /// This prevents the vma from being expanded with `mremap()`. + #[inline] + pub fn set_dontexpand(&self) { + // SAFETY: Setting the VM_DONTEXPAND flag is always okay. + unsafe { self.update_flags(flags::DONTEXPAND, 0) }; + } + + /// Set the `VM_DONTCOPY` flag on this vma. + /// + /// This prevents the vma from being copied on fork. This option is only permanent if `VM_IO` + /// is set. + #[inline] + pub fn set_dontcopy(&self) { + // SAFETY: Setting the VM_DONTCOPY flag is always okay. + unsafe { self.update_flags(flags::DONTCOPY, 0) }; + } + + /// Set the `VM_DONTDUMP` flag on this vma. + /// + /// This prevents the vma from being included in core dumps. This option is only permanent if + /// `VM_IO` is set. + #[inline] + pub fn set_dontdump(&self) { + // SAFETY: Setting the VM_DONTDUMP flag is always okay. + unsafe { self.update_flags(flags::DONTDUMP, 0) }; + } + + /// Returns whether `VM_READ` is set. + /// + /// This flag indicates whether userspace is mapping this vma as readable. + #[inline] + pub fn readable(&self) -> bool { + (self.flags() & flags::READ) != 0 + } + + /// Try to clear the `VM_MAYREAD` flag, failing if `VM_READ` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma readable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYREAD` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_mayread(&self) -> Result { + if self.readable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYREAD` is okay when `VM_READ` is not set. + unsafe { self.update_flags(0, flags::MAYREAD) }; + Ok(()) + } + + /// Returns whether `VM_WRITE` is set. + /// + /// This flag indicates whether userspace is mapping this vma as writable. + #[inline] + pub fn writable(&self) -> bool { + (self.flags() & flags::WRITE) != 0 + } + + /// Try to clear the `VM_MAYWRITE` flag, failing if `VM_WRITE` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma writable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYWRITE` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_maywrite(&self) -> Result { + if self.writable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYWRITE` is okay when `VM_WRITE` is not set. + unsafe { self.update_flags(0, flags::MAYWRITE) }; + Ok(()) + } + + /// Returns whether `VM_EXEC` is set. + /// + /// This flag indicates whether userspace is mapping this vma as executable. + #[inline] + pub fn executable(&self) -> bool { + (self.flags() & flags::EXEC) != 0 + } + + /// Try to clear the `VM_MAYEXEC` flag, failing if `VM_EXEC` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma executable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYEXEC` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_mayexec(&self) -> Result { + if self.executable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYEXEC` is okay when `VM_EXEC` is not set. + unsafe { self.update_flags(0, flags::MAYEXEC) }; + Ok(()) + } +} + +/// The integer type used for vma flags. +#[doc(inline)] +pub use bindings::vm_flags_t; + +/// All possible flags for [`VmaRef`]. +pub mod flags { + use super::vm_flags_t; + use crate::bindings; + + /// No flags are set. + pub const NONE: vm_flags_t = bindings::VM_NONE as vm_flags_t; + + /// Mapping allows reads. + pub const READ: vm_flags_t = bindings::VM_READ as vm_flags_t; + + /// Mapping allows writes. + pub const WRITE: vm_flags_t = bindings::VM_WRITE as vm_flags_t; + + /// Mapping allows execution. + pub const EXEC: vm_flags_t = bindings::VM_EXEC as vm_flags_t; + + /// Mapping is shared. + pub const SHARED: vm_flags_t = bindings::VM_SHARED as vm_flags_t; + + /// Mapping may be updated to allow reads. + pub const MAYREAD: vm_flags_t = bindings::VM_MAYREAD as vm_flags_t; + + /// Mapping may be updated to allow writes. + pub const MAYWRITE: vm_flags_t = bindings::VM_MAYWRITE as vm_flags_t; + + /// Mapping may be updated to allow execution. + pub const MAYEXEC: vm_flags_t = bindings::VM_MAYEXEC as vm_flags_t; + + /// Mapping may be updated to be shared. + pub const MAYSHARE: vm_flags_t = bindings::VM_MAYSHARE as vm_flags_t; + + /// Page-ranges managed without `struct page`, just pure PFN. + pub const PFNMAP: vm_flags_t = bindings::VM_PFNMAP as vm_flags_t; + + /// Memory mapped I/O or similar. + pub const IO: vm_flags_t = bindings::VM_IO as vm_flags_t; + + /// Do not copy this vma on fork. + pub const DONTCOPY: vm_flags_t = bindings::VM_DONTCOPY as vm_flags_t; + + /// Cannot expand with mremap(). + pub const DONTEXPAND: vm_flags_t = bindings::VM_DONTEXPAND as vm_flags_t; + + /// Lock the pages covered when they are faulted in. + pub const LOCKONFAULT: vm_flags_t = bindings::VM_LOCKONFAULT as vm_flags_t; + + /// Is a VM accounted object. + pub const ACCOUNT: vm_flags_t = bindings::VM_ACCOUNT as vm_flags_t; + + /// Should the VM suppress accounting. + pub const NORESERVE: vm_flags_t = bindings::VM_NORESERVE as vm_flags_t; + + /// Huge TLB Page VM. + pub const HUGETLB: vm_flags_t = bindings::VM_HUGETLB as vm_flags_t; + + /// Synchronous page faults. (DAX-specific) + pub const SYNC: vm_flags_t = bindings::VM_SYNC as vm_flags_t; + + /// Architecture-specific flag. + pub const ARCH_1: vm_flags_t = bindings::VM_ARCH_1 as vm_flags_t; + + /// Wipe VMA contents in child on fork. + pub const WIPEONFORK: vm_flags_t = bindings::VM_WIPEONFORK as vm_flags_t; + + /// Do not include in the core dump. + pub const DONTDUMP: vm_flags_t = bindings::VM_DONTDUMP as vm_flags_t; + + /// Not soft dirty clean area. + pub const SOFTDIRTY: vm_flags_t = bindings::VM_SOFTDIRTY as vm_flags_t; + + /// Can contain `struct page` and pure PFN pages. + pub const MIXEDMAP: vm_flags_t = bindings::VM_MIXEDMAP as vm_flags_t; + + /// MADV_HUGEPAGE marked this vma. + pub const HUGEPAGE: vm_flags_t = bindings::VM_HUGEPAGE as vm_flags_t; + + /// MADV_NOHUGEPAGE marked this vma. + pub const NOHUGEPAGE: vm_flags_t = bindings::VM_NOHUGEPAGE as vm_flags_t; + + /// KSM may merge identical pages. + pub const MERGEABLE: vm_flags_t = bindings::VM_MERGEABLE as vm_flags_t; +} diff --git a/rust/kernel/module.rs b/rust/kernel/module.rs new file mode 100644 index 000000000..d71370598 --- /dev/null +++ b/rust/kernel/module.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Module-related types and helpers. + +/// The entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync + Send { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> crate::error::Result; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl pin_init::PinInit; +} + +impl InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit { + let initer = move |slot: *mut Self| { + let m = ::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { pin_init::pin_init_from_closure(initer) } + } +} + +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; + + /// The module's `THIS_MODULE` pointer. + const THIS_MODULE: ThisModule; +} + +/// Returns a reference to the `THIS_MODULE` of the given module type. +#[inline] +pub const fn this_module() -> &'static ThisModule { + &M::THIS_MODULE +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) +pub struct ThisModule(*mut crate::bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut crate::bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut crate::bindings::module { + self.0 + } +} diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 000000000..f9a14765a --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) + +use crate::prelude::*; +use crate::str::{kstrtobool_bytes, BStr}; +use bindings; +use kernel::sync::SetOnce; + +/// Newtype to make `bindings::kernel_param` [`Sync`]. +#[repr(transparent)] +#[doc(hidden)] +pub struct KernelParam(bindings::kernel_param); + +impl KernelParam { + #[doc(hidden)] + pub const fn new(val: bindings::kernel_param) -> Self { + Self(val) + } +} + +// SAFETY: C kernel handles serializing access to this type. We never access it +// from Rust module. +unsafe impl Sync for KernelParam {} + +/// Types that can be used for module parameters. +// NOTE: This trait is `Copy` because drop could produce unsoundness during teardown. +pub trait ModuleParam: Sized + Copy { + /// Parse a parameter argument into the parameter value. + fn try_from_param_arg(arg: &BStr) -> Result; +} + +/// Set the module parameter from a string. +/// +/// Used to set the parameter value at kernel initialization, when loading +/// the module or when set through `sysfs`. +/// +/// See `struct kernel_param_ops.set`. +/// +/// # Safety +/// +/// - If `val` is non-null then it must point to a valid null-terminated string that must be valid +/// for reads for the duration of the call. +/// - `param` must be a pointer to a `bindings::kernel_param` initialized by the rust module macro. +/// The pointee must be valid for reads for the duration of the call. +/// +/// # Note +/// +/// - The safety requirements are satisfied by C API contract when this function is invoked by the +/// module subsystem C code. +/// - Currently, we only support read-only parameters that are not readable from `sysfs`. Thus, this +/// function is only called at kernel initialization time, or at module load time, and we have +/// exclusive access to the parameter for the duration of the function. +/// +/// [`module!`]: macros::module +unsafe extern "C" fn set_param(val: *const c_char, param: *const bindings::kernel_param) -> c_int +where + T: ModuleParam, +{ + // NOTE: If we start supporting arguments without values, val _is_ allowed + // to be null here. + if val.is_null() { + crate::pr_warn_once!("Null pointer passed to `module_param::set_param`\n"); + return EINVAL.to_errno(); + } + + // SAFETY: By function safety requirement, val is non-null, null-terminated + // and valid for reads for the duration of this function. + let arg = unsafe { CStr::from_char_ptr(val) }; + let arg: &BStr = arg.as_ref(); + + crate::error::from_result(|| { + let new_value = T::try_from_param_arg(arg)?; + + // SAFETY: By function safety requirements, this access is safe. + let container = unsafe { &*((*param).__bindgen_anon_1.arg.cast::>()) }; + + container + .populate(new_value) + .then_some(0) + .ok_or(kernel::error::code::EEXIST) + }) +} + +macro_rules! impl_int_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + fn try_from_param_arg(arg: &BStr) -> Result { + <$ty as crate::str::parse_int::ParseInt>::from_str(arg) + } + } + }; +} + +impl_int_module_param!(i8); +impl_int_module_param!(u8); +impl_int_module_param!(i16); +impl_int_module_param!(u16); +impl_int_module_param!(i32); +impl_int_module_param!(u32); +impl_int_module_param!(i64); +impl_int_module_param!(u64); +impl_int_module_param!(isize); +impl_int_module_param!(usize); + +impl ModuleParam for bool { + fn try_from_param_arg(arg: &BStr) -> Result { + kstrtobool_bytes(arg) + } +} + +/// A wrapper for kernel parameters. +/// +/// This type is instantiated by the [`module!`] macro when module parameters are +/// defined. You should never need to instantiate this type directly. +/// +/// Note: This type is `pub` because it is used by module crates to access +/// parameter values. +pub struct ModuleParamAccess { + value: SetOnce, + default: T, +} + +// SAFETY: We only create shared references to the contents of this container, +// so if `T` is `Sync`, so is `ModuleParamAccess`. +unsafe impl Sync for ModuleParamAccess {} + +impl ModuleParamAccess { + #[doc(hidden)] + pub const fn new(default: T) -> Self { + Self { + value: SetOnce::new(), + default, + } + } + + /// Get a copy of the parameter value. + /// + /// Returns the value supplied at module load time, or the default value + /// if the parameter has not been set. + #[inline] + pub fn value(&self) -> T + where + T: Copy, + { + self.value.copy().unwrap_or(self.default) + } + + /// Get a shared reference to the parameter value. + /// + /// Returns a reference to the value supplied at module load time, or a + /// reference to the default value if the parameter has not been set. + // Note: When sysfs access to parameters are enabled, we have to pass in a + // held lock guard here. + #[inline] + pub fn value_ref(&self) -> &T { + self.value.as_ref().unwrap_or(&self.default) + } + + /// Get a mutable pointer to `self`. + /// + /// NOTE: In most cases it is not safe deref the returned pointer. + pub const fn as_void_ptr(&self) -> *mut c_void { + core::ptr::from_ref(self).cast_mut().cast() + } +} + +#[doc(hidden)] +/// Generate a static [`kernel_param_ops`](srctree/include/linux/moduleparam.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + #[doc(hidden)] + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: 0, + set: Some(set_param::<$ty>), + get: None, + free: None, + }; + }; +} + +make_param_ops!(PARAM_OPS_I8, i8); +make_param_ops!(PARAM_OPS_U8, u8); +make_param_ops!(PARAM_OPS_I16, i16); +make_param_ops!(PARAM_OPS_U16, u16); +make_param_ops!(PARAM_OPS_I32, i32); +make_param_ops!(PARAM_OPS_U32, u32); +make_param_ops!(PARAM_OPS_I64, i64); +make_param_ops!(PARAM_OPS_U64, u64); +make_param_ops!(PARAM_OPS_ISIZE, isize); +make_param_ops!(PARAM_OPS_USIZE, usize); +make_param_ops!(PARAM_OPS_BOOL, bool); diff --git a/rust/kernel/net/mod.rs b/rust/kernel/net/mod.rs new file mode 100644 index 000000000..8ecae7577 --- /dev/null +++ b/rust/kernel/net/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Networking. + +#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] +pub mod phy; + +pub mod netlink; diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs new file mode 100644 index 000000000..22ef3dde3 --- /dev/null +++ b/rust/kernel/net/netlink.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2026 Google LLC. + +//! Rust support for generic netlink. +//! +//! Currently only supports exposing multicast groups. +//! +//! C header: [`include/net/genetlink.h`](srctree/include/net/genetlink.h) + +use kernel::{ + alloc::{self, AllocError}, + error::to_result, + prelude::*, + transmute::AsBytes, + types::Opaque, + ThisModule, +}; + +use core::{ + mem::ManuallyDrop, + ptr::NonNull, // +}; + +/// The default netlink message size. +pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE; + +/// A wrapper around `struct sk_buff` for generic netlink messages. +/// +/// This type is intended to be specific for buffers used with netlink only, and other usecases for +/// `struct sk_buff` are out-of-scope for this abstraction. +/// +/// # Invariants +/// +/// The pointer has ownership over a valid `sk_buff`. +pub struct NetlinkSkBuff { + skb: NonNull, +} + +impl NetlinkSkBuff { + /// Creates a new `NetlinkSkBuff` with the given size. + pub fn new(size: usize, flags: alloc::Flags) -> Result { + // SAFETY: `genlmsg_new` only requires its arguments to be valid integers. + let skb = unsafe { bindings::genlmsg_new(size, flags.as_raw()) }; + let skb = NonNull::new(skb).ok_or(AllocError)?; + Ok(NetlinkSkBuff { skb }) + } + + /// Puts a generic netlink header into the `NetlinkSkBuff`. + pub fn genlmsg_put( + self, + portid: u32, + seq: u32, + family: &'static Family, + cmd: u8, + ) -> Result { + let skb = self.skb.as_ptr(); + // SAFETY: The skb and family pointers are valid. + let hdr = unsafe { bindings::genlmsg_put(skb, portid, seq, family.as_raw(), 0, cmd) }; + let hdr = NonNull::new(hdr).ok_or(AllocError)?; + Ok(GenlMsg { skb: self, hdr }) + } +} + +impl Drop for NetlinkSkBuff { + fn drop(&mut self) { + // SAFETY: We have ownership over the `sk_buff`, so we may free it. + unsafe { bindings::nlmsg_free(self.skb.as_ptr()) } + } +} + +/// A generic netlink message being constructed. +/// +/// # Invariants +/// +/// `hdr` references the header in this netlink message. +pub struct GenlMsg { + skb: NetlinkSkBuff, + hdr: NonNull, +} + +impl GenlMsg { + /// Puts an attribute into the message. + #[inline] + fn put(&mut self, attrtype: c_int, value: &T) -> Result + where + T: ?Sized + AsBytes, + { + let skb = self.skb.skb.as_ptr(); + let len = size_of_val(value); + let ptr = core::ptr::from_ref(value).cast::(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is + // readable and initialized for its `size_of` bytes. + to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) + } + + /// Puts a `u32` attribute into the message. + #[inline] + pub fn put_u32(&mut self, attrtype: c_int, value: u32) -> Result { + self.put(attrtype, &value) + } + + /// Puts a string attribute into the message. + #[inline] + pub fn put_string(&mut self, attrtype: c_int, value: &CStr) -> Result { + self.put(attrtype, value.to_bytes_with_nul()) + } + + /// Puts a flag attribute into the message. + #[inline] + pub fn put_flag(&mut self, attrtype: c_int) -> Result { + let skb = self.skb.skb.as_ptr(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and a null pointer is valid + // when the length is zero. + to_result(unsafe { bindings::nla_put(skb, attrtype, 0, core::ptr::null()) }) + } + + /// Sends the generic netlink message as a multicast message. + #[inline] + pub fn multicast( + self, + family: &'static Family, + portid: u32, + group: u32, + flags: alloc::Flags, + ) -> Result { + let me = ManuallyDrop::new(self); + // SAFETY: The `skb` and `family` pointers are valid. We pass ownership of the `skb` to + // `genlmsg_multicast` by not dropping `self`. + unsafe { + bindings::genlmsg_end(me.skb.skb.as_ptr(), me.hdr.as_ptr()); + to_result(bindings::genlmsg_multicast( + family.as_raw(), + me.skb.skb.as_ptr(), + portid, + group, + flags.as_raw(), + )) + } + } +} +impl Drop for GenlMsg { + fn drop(&mut self) { + // SAFETY: The `hdr` pointer references the header of this generic netlink message. + unsafe { bindings::genlmsg_cancel(self.skb.skb.as_ptr(), self.hdr.as_ptr()) }; + } +} + +/// Flags for a generic netlink family. +struct FamilyFlags { + /// Whether the family supports network namespaces. + netnsok: bool, + /// Whether the family supports parallel operations. + parallel_ops: bool, +} + +impl FamilyFlags { + /// Converts the flags to the bitfield representation used by `genl_family`. + const fn into_bitfield(self) -> bindings::__BindgenBitfieldUnit<[u8; 1]> { + // The below shifts are verified correct by test_family_flags_bitfield() below. + // + // Although bindgen generates helpers to change bitfields based on the C headers, these + // helpers unfortunately can't be used in const context. Since `Family` needs to be filled + // out at build-time, we use this helper instead. + let mut bits = 0; + if self.netnsok { + bits |= 1 << 0; + } + if self.parallel_ops { + bits |= 1 << 1; + } + // Convert from little endian to the target's endianness. + bits = u8::from_le(bits); + // SAFETY: This bitfield is represented as an u8. + unsafe { core::mem::transmute::>(bits) } + } +} + +/// A generic netlink family. +#[repr(transparent)] +pub struct Family { + inner: Opaque, +} + +// SAFETY: The `Family` type is thread safe. +unsafe impl Sync for Family {} + +impl Family { + /// Creates a new `Family` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + /// + /// The name must be a nul-terminated string, but it is taken as `&[u8]` so that it can be used + /// more conveniently with the strings generated by bindgen. + pub const fn const_new( + module: &ThisModule, + name: &[u8], + version: u32, + mcgrps: &'static [MulticastGroup], + ) -> Family { + let n_mcgrps = mcgrps.len() as u8; + if n_mcgrps as usize != mcgrps.len() { + panic!("too many mcgrps"); + } + let mut genl_family = bindings::genl_family { + version, + _bitfield_1: FamilyFlags { + netnsok: true, + parallel_ops: true, + } + .into_bitfield(), + module: module.as_ptr(), + mcgrps: mcgrps.as_ptr().cast(), + n_mcgrps, + ..pin_init::zeroed() + }; + if CStr::from_bytes_with_nul(name).is_err() { + panic!("genl_family name not nul-terminated"); + } + if genl_family.name.len() < name.len() { + panic!("genl_family name too long"); + } + let mut i = 0; + while i < name.len() { + genl_family.name[i] = name[i]; + i += 1; + } + Family { + inner: Opaque::new(genl_family), + } + } + + /// Checks if there are any listeners for the given multicast group. + pub fn has_listeners(&self, group: u32) -> bool { + // SAFETY: The family and init_net pointers are valid. + unsafe { + bindings::genl_has_listeners(self.as_raw(), &raw mut bindings::init_net, group) != 0 + } + } + + /// Returns a raw pointer to the underlying `genl_family` structure. + pub fn as_raw(&self) -> *mut bindings::genl_family { + self.inner.get() + } +} + +/// A generic netlink multicast group. +#[repr(transparent)] +pub struct MulticastGroup { + // No Opaque because fully immutable + group: bindings::genl_multicast_group, +} + +// SAFETY: Pure data so thread safe. +unsafe impl Sync for MulticastGroup {} + +impl MulticastGroup { + /// Creates a new `MulticastGroup` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + pub const fn const_new(name: &CStr) -> MulticastGroup { + let mut group: bindings::genl_multicast_group = pin_init::zeroed(); + + let name = name.to_bytes_with_nul(); + if group.name.len() < name.len() { + panic!("genl_multicast_group name too long"); + } + let mut i = 0; + while i < name.len() { + group.name[i] = name[i]; + i += 1; + } + + MulticastGroup { group } + } +} + +/// A registration of a generic netlink family. +/// +/// This type represents the registration of a [`Family`]. When an instance of this type is +/// dropped, its respective generic netlink family will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.family` always holds a valid reference to an initialized and registered [`Family`]. +pub struct Registration { + family: &'static Family, +} + +impl Family { + /// Registers the generic netlink family with the kernel. + pub fn register(&'static self) -> Result { + // SAFETY: `self.as_raw()` is a valid pointer to a `genl_family` struct. + // The `genl_family` struct is static, so it will outlive the registration. + to_result(unsafe { bindings::genl_register_family(self.as_raw()) })?; + Ok(Registration { family: self }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `self.family.as_raw()` is a valid pointer to a registered `genl_family` struct. + // The `Registration` struct ensures that `genl_unregister_family` is called exactly once + // for this family when it goes out of scope. + unsafe { bindings::genl_unregister_family(self.family.as_raw()) }; + } +} + +#[macros::kunit_tests(rust_netlink)] +mod tests { + use super::*; + + #[test] + fn test_family_flags_bitfield() { + for netnsok in [false, true] { + for parallel_ops in [false, true] { + let mut b_fam = bindings::genl_family { + ..Default::default() + }; + b_fam.set_netnsok(if netnsok { 1 } else { 0 }); + b_fam.set_parallel_ops(if parallel_ops { 1 } else { 0 }); + + let c_bitfield = FamilyFlags { + netnsok, + parallel_ops, + } + .into_bitfield(); + + // SAFETY: The bit field is stored as u8. + let b_val: u8 = unsafe { core::mem::transmute(b_fam._bitfield_1) }; + // SAFETY: The bit field is stored as u8. + let c_val: u8 = unsafe { core::mem::transmute(c_bitfield) }; + assert_eq!(b_val, c_val); + } + } + } +} diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs new file mode 100644 index 000000000..956cda573 --- /dev/null +++ b/rust/kernel/net/phy.rs @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2023 FUJITA Tomonori + +//! Network PHY device. +//! +//! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). + +use crate::{device_id::RawDeviceId, error::*, prelude::*, types::Opaque}; +use core::{marker::PhantomData, ptr::addr_of_mut}; + +pub mod reg; + +/// PHY state machine states. +/// +/// Corresponds to the kernel's [`enum phy_state`]. +/// +/// Some of PHY drivers access to the state of PHY's software state machine. +/// +/// [`enum phy_state`]: srctree/include/linux/phy.h +#[derive(PartialEq, Eq)] +pub enum DeviceState { + /// PHY device and driver are not ready for anything. + Down, + /// PHY is ready to send and receive packets. + Ready, + /// PHY is up, but no polling or interrupts are done. + Halted, + /// PHY is up, but is in an error state. + Error, + /// PHY and attached device are ready to do work. + Up, + /// PHY is currently running. + Running, + /// PHY is up, but not currently plugged in. + NoLink, + /// PHY is performing a cable test. + CableTest, +} + +/// A mode of Ethernet communication. +/// +/// PHY drivers get duplex information from hardware and update the current state. +pub enum DuplexMode { + /// PHY is in full-duplex mode. + Full, + /// PHY is in half-duplex mode. + Half, + /// PHY is in unknown duplex mode. + Unknown, +} + +/// An instance of a PHY device. +/// +/// Wraps the kernel's [`struct phy_device`]. +/// +/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver +/// executes [`Driver`]'s methods during the callback. +/// +/// # Invariants +/// +/// - Referencing a `phy_device` using this struct asserts that you are in +/// a context where all methods defined on this struct are safe to call. +/// - This struct always has a valid `self.0.mdio.dev`. +/// +/// [`struct phy_device`]: srctree/include/linux/phy.h +// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is +// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for +// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with +// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance. +// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access +// to the instance. +#[repr(transparent)] +pub struct Device(Opaque); + +impl Device { + /// Creates a new [`Device`] instance from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, + /// - the pointer must point at a valid `phy_device`, and the caller + /// must be in a context where all methods defined on this struct + /// are safe to call. + /// - `(*ptr).mdio.dev` must be a valid. + unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self { + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`. + let ptr = ptr.cast::(); + // SAFETY: by the function requirements the pointer is valid and we have unique access for + // the duration of `'a`. + unsafe { &mut *ptr } + } + + /// Gets the id of the PHY. + pub fn phy_id(&self) -> u32 { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).phy_id } + } + + /// Gets the state of PHY state machine states. + pub fn state(&self) -> DeviceState { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let state = unsafe { (*phydev).state }; + // TODO: this conversion code will be replaced with automatically generated code by bindgen + // when it becomes possible. + match state { + bindings::phy_state_PHY_DOWN => DeviceState::Down, + bindings::phy_state_PHY_READY => DeviceState::Ready, + bindings::phy_state_PHY_HALTED => DeviceState::Halted, + bindings::phy_state_PHY_ERROR => DeviceState::Error, + bindings::phy_state_PHY_UP => DeviceState::Up, + bindings::phy_state_PHY_RUNNING => DeviceState::Running, + bindings::phy_state_PHY_NOLINK => DeviceState::NoLink, + bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest, + _ => DeviceState::Error, + } + } + + /// Gets the current link state. + /// + /// It returns true if the link is up. + pub fn is_link_up(&self) -> bool { + const LINK_IS_UP: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(14, 1) == LINK_IS_UP + } + + /// Gets the current auto-negotiation configuration. + /// + /// It returns true if auto-negotiation is enabled. + pub fn is_autoneg_enabled(&self) -> bool { + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(13, 1) == u64::from(bindings::AUTONEG_ENABLE) + } + + /// Gets the current auto-negotiation state. + /// + /// It returns true if auto-negotiation is completed. + pub fn is_autoneg_completed(&self) -> bool { + const AUTONEG_COMPLETED: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(15, 1) == AUTONEG_COMPLETED + } + + /// Sets the speed of the PHY. + pub fn set_speed(&mut self, speed: u32) { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).speed = speed as c_int }; + } + + /// Sets duplex mode. + pub fn set_duplex(&mut self, mode: DuplexMode) { + let phydev = self.0.get(); + let v = match mode { + DuplexMode::Full => bindings::DUPLEX_FULL, + DuplexMode::Half => bindings::DUPLEX_HALF, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN, + }; + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).duplex = v as c_int }; + } + + /// Reads a PHY register. + // This function reads a hardware register and updates the stats so takes `&mut self`. + pub fn read(&mut self, reg: R) -> Result { + reg.read(self) + } + + /// Writes a PHY register. + pub fn write(&mut self, reg: R, val: u16) -> Result { + reg.write(self, val) + } + + /// Reads a paged register. + pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) }; + + to_result(ret).map(|()| ret as u16) + } + + /// Resolves the advertisements into PHY settings. + pub fn resolve_aneg_linkmode(&mut self) { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + unsafe { bindings::phy_resolve_aneg_linkmode(phydev) }; + } + + /// Executes software reset the PHY via `BMCR_RESET` bit. + pub fn genphy_soft_reset(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_soft_reset(phydev) }) + } + + /// Initializes the PHY. + pub fn init_hw(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::phy_init_hw(phydev) }) + } + + /// Starts auto-negotiation. + pub fn start_aneg(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::_phy_start_aneg(phydev) }) + } + + /// Resumes the PHY via `BMCR_PDOWN` bit. + pub fn genphy_resume(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_resume(phydev) }) + } + + /// Suspends the PHY via `BMCR_PDOWN` bit. + pub fn genphy_suspend(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_suspend(phydev) }) + } + + /// Checks the link status and updates current link state. + pub fn genphy_read_status(&mut self) -> Result { + R::read_status(self) + } + + /// Updates the link status. + pub fn genphy_update_link(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_update_link(phydev) }) + } + + /// Reads link partner ability. + pub fn genphy_read_lpa(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_lpa(phydev) }) + } + + /// Reads PHY abilities. + pub fn genphy_read_abilities(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_abilities(phydev) }) + } +} + +impl AsRef for Device { + fn as_ref(&self) -> &kernel::device::Device { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that `mdio.dev` is valid. + unsafe { kernel::device::Device::from_raw(addr_of_mut!((*phydev).mdio.dev)) } + } +} + +/// Defines certain other features this PHY supports (like interrupts). +/// +/// These flag values are used in [`Driver::FLAGS`]. +pub mod flags { + /// PHY is internal. + pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL; + /// PHY needs to be reset after the refclk is enabled. + pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN; + /// Polling is used to detect PHY status changes. + pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST; + /// Don't suspend. + pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND; +} + +/// An adapter for the registration of a PHY driver. +struct Adapter { + _p: PhantomData, +} + +impl Adapter { + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn soft_reset_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::soft_reset(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we can exclusively access `phy_device` because + // it's not published yet, so the accessors on `Device` are okay + // to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::probe(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn get_features_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::get_features(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::suspend(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::resume(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn config_aneg_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::config_aneg(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_status_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::read_status(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn match_phy_device_callback( + phydev: *mut bindings::phy_device, + _phydrv: *const bindings::phy_driver, + ) -> c_int { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::match_phy_device(dev).into() + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + // CAST: the C side verifies devnum < 32. + let ret = T::read_mmd(dev, devnum as u8, regnum)?; + Ok(ret.into()) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn write_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + val: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::write_mmd(dev, devnum as u8, regnum, val)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::link_change_notify(dev); + } +} + +/// Driver structure for a particular PHY type. +/// +/// Wraps the kernel's [`struct phy_driver`]. +/// This is used to register a driver for a particular PHY type with the kernel. +/// +/// # Invariants +/// +/// `self.0` is always in a valid state. +/// +/// [`struct phy_driver`]: srctree/include/linux/phy.h +#[repr(transparent)] +pub struct DriverVTable(Opaque); + +// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to +// share `&DriverVTable` across execution context boundaries. +unsafe impl Sync for DriverVTable {} + +/// Creates a [`DriverVTable`] instance from [`Driver`]. +/// +/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`. +/// +/// [`module_phy_driver`]: crate::module_phy_driver +pub const fn create_phy_driver() -> DriverVTable { + // INVARIANT: All the fields of `struct phy_driver` are initialized properly. + DriverVTable(Opaque::new(bindings::phy_driver { + name: crate::str::as_char_ptr_in_const_context(T::NAME).cast_mut(), + flags: T::FLAGS, + phy_id: T::PHY_DEVICE_ID.id(), + phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), + soft_reset: if T::HAS_SOFT_RESET { + Some(Adapter::::soft_reset_callback) + } else { + None + }, + probe: if T::HAS_PROBE { + Some(Adapter::::probe_callback) + } else { + None + }, + get_features: if T::HAS_GET_FEATURES { + Some(Adapter::::get_features_callback) + } else { + None + }, + match_phy_device: if T::HAS_MATCH_PHY_DEVICE { + Some(Adapter::::match_phy_device_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Adapter::::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Adapter::::resume_callback) + } else { + None + }, + config_aneg: if T::HAS_CONFIG_ANEG { + Some(Adapter::::config_aneg_callback) + } else { + None + }, + read_status: if T::HAS_READ_STATUS { + Some(Adapter::::read_status_callback) + } else { + None + }, + read_mmd: if T::HAS_READ_MMD { + Some(Adapter::::read_mmd_callback) + } else { + None + }, + write_mmd: if T::HAS_WRITE_MMD { + Some(Adapter::::write_mmd_callback) + } else { + None + }, + link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY { + Some(Adapter::::link_change_notify_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct phy_driver`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::::zeroed().assume_init() } + })) +} + +/// Driver implementation for a particular PHY type. +/// +/// This trait is used to create a [`DriverVTable`]. +#[vtable] +pub trait Driver { + /// Defines certain other features this PHY supports. + /// It is a combination of the flags in the [`flags`] module. + const FLAGS: u32 = 0; + + /// The friendly name of this PHY type. + const NAME: &'static CStr; + + /// This driver only works for PHYs with IDs which match this field. + /// The default id and mask are zero. + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0); + + /// Issues a PHY software reset. + fn soft_reset(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Sets up device-specific structures during discovery. + fn probe(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Probes the hardware to determine what abilities it has. + fn get_features(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Returns true if this is a suitable driver for the given phydev. + /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`]. + fn match_phy_device(_dev: &Device) -> bool { + false + } + + /// Configures the advertisement and resets auto-negotiation + /// if auto-negotiation is enabled. + fn config_aneg(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Determines the negotiated speed and duplex. + fn read_status(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Suspends the hardware, saving state if needed. + fn suspend(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Resumes the hardware, restoring state if needed. + fn resume(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD read function for reading a MMD register. + fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD write function for writing a MMD register. + fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Callback for notification of link change. + fn link_change_notify(_dev: &mut Device) {} +} + +/// Registration structure for PHY drivers. +/// +/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped. +/// +/// # Invariants +/// +/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`. +pub struct Registration { + drivers: Pin<&'static mut [DriverVTable]>, +} + +// SAFETY: The only action allowed in a `Registration` instance is dropping it, which is safe to do +// from any thread because `phy_drivers_unregister` can be called from any thread context. +unsafe impl Send for Registration {} + +impl Registration { + /// Registers a PHY driver. + pub fn register( + module: &'static crate::ThisModule, + drivers: Pin<&'static mut [DriverVTable]>, + ) -> Result { + if drivers.is_empty() { + return Err(code::EINVAL); + } + // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of + // the `drivers` slice are initialized properly. `drivers` will not be moved. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_drivers_register( + drivers[0].0.get(), + drivers.len().try_into()?, + module.as_ptr(), + ) + })?; + // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. + Ok(Registration { drivers }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.drivers` is valid. + // So it's just an FFI call. + unsafe { + bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32) + }; + } +} + +/// An identifier for PHY devices on an MDIO/MII bus. +/// +/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate +/// PHY driver. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::mdio_device_id); + +impl DeviceId { + /// Creates a new instance with the exact match mask. + pub const fn new_with_exact_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Exact.as_int(), + }) + } + + /// Creates a new instance with the model match mask. + pub const fn new_with_model_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Model.as_int(), + }) + } + + /// Creates a new instance with the vendor match mask. + pub const fn new_with_vendor_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Vendor.as_int(), + }) + } + + /// Creates a new instance with a custom match mask. + pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Custom(mask).as_int(), + }) + } + + /// Creates a new instance from [`Driver`]. + pub const fn new_with_driver() -> Self { + T::PHY_DEVICE_ID + } + + /// Get the MDIO device's PHY ID. + pub const fn id(&self) -> u32 { + self.0.phy_id + } + + /// Get the MDIO device's match mask. + pub const fn mask_as_int(&self) -> u32 { + self.0.phy_id_mask + } + + // macro use only + #[doc(hidden)] + pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { + self.0 + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct mdio_device_id` +// and does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::mdio_device_id; +} + +enum DeviceMask { + Exact, + Model, + Vendor, + Custom(u32), +} + +impl DeviceMask { + const MASK_EXACT: u32 = !0; + const MASK_MODEL: u32 = !0 << 4; + const MASK_VENDOR: u32 = !0 << 10; + + const fn as_int(&self) -> u32 { + match self { + DeviceMask::Exact => Self::MASK_EXACT, + DeviceMask::Model => Self::MASK_MODEL, + DeviceMask::Vendor => Self::MASK_VENDOR, + DeviceMask::Custom(mask) => *mask, + } + } +} + +/// Declares a kernel module for PHYs drivers. +/// +/// This creates a static array of kernel's `struct phy_driver` and registers it. +/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information +/// for module loading into the module binary file. Every driver needs an entry in `device_table`. +/// +/// # Examples +/// +/// ``` +/// # mod module_phy_driver_sample { +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// kernel::module_phy_driver! { +/// drivers: [PhySample], +/// device_table: [ +/// DeviceId::new_with_driver::() +/// ], +/// name: "rust_sample_phy", +/// authors: ["Rust for Linux Contributors"], +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c"PhySample"; +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// # } +/// ``` +#[macro_export] +macro_rules! module_phy_driver { + (@replace_expr $_t:tt $sub:expr) => {$sub}; + + (@count_devices $($x:expr),*) => { + 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))* + }; + + (@device_table [$($dev:expr),+]) => { + $crate::module_device_table!( + "mdio", $crate::net::phy::DeviceId, + TABLE, @none, [$($dev),+] + ); + }; + + (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { + struct Module { + _reg: $crate::net::phy::Registration, + } + + $crate::prelude::module! { + type: Module, + $($f)* + } + + const _: () = { + static mut DRIVERS: [$crate::net::phy::DriverVTable; + $crate::module_phy_driver!(@count_devices $($driver),+)] = + [$($crate::net::phy::create_phy_driver::<$driver>()),+]; + + impl $crate::Module for Module { + fn init(module: &'static $crate::ThisModule) -> Result { + // SAFETY: The anonymous constant guarantees that nobody else can access + // the `DRIVERS` static. The array is used only in the C side. + let drivers = unsafe { &mut DRIVERS }; + let mut reg = $crate::net::phy::Registration::register( + module, + ::core::pin::Pin::static_mut(drivers), + )?; + Ok(Module { _reg: reg }) + } + } + }; + + $crate::module_phy_driver!(@device_table [$($dev),+]); + } +} diff --git a/rust/kernel/net/phy/reg.rs b/rust/kernel/net/phy/reg.rs new file mode 100644 index 000000000..80e22c264 --- /dev/null +++ b/rust/kernel/net/phy/reg.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 FUJITA Tomonori + +//! PHY register interfaces. +//! +//! This module provides support for accessing PHY registers in the +//! Ethernet management interface clauses 22 and 45 register namespaces, as +//! defined in IEEE 802.3. + +use super::Device; +use crate::{ + build_assert::build_assert, + error::*, + uapi, // +}; + +mod private { + /// Marker that a trait cannot be implemented outside of this crate + pub trait Sealed {} +} + +/// Accesses PHY registers. +/// +/// This trait is used to implement the unified interface to access +/// C22 and C45 PHY registers. +/// +/// # Examples +/// +/// ```ignore +/// fn link_change_notify(dev: &mut Device) { +/// // read C22 BMCR register +/// dev.read(C22::BMCR); +/// // read C45 PMA/PMD control 1 register +/// dev.read(C45::new(Mmd::PMAPMD, 0)); +/// +/// // Checks the link status as reported by registers in the C22 namespace +/// // and updates current link state. +/// dev.genphy_read_status::(); +/// // Checks the link status as reported by registers in the C45 namespace +/// // and updates current link state. +/// dev.genphy_read_status::(); +/// } +/// ``` +pub trait Register: private::Sealed { + /// Reads a PHY register. + fn read(&self, dev: &mut Device) -> Result; + + /// Writes a PHY register. + fn write(&self, dev: &mut Device, val: u16) -> Result; + + /// Checks the link status and updates current link state. + fn read_status(dev: &mut Device) -> Result; +} + +/// A single MDIO clause 22 register address (5 bits). +#[derive(Copy, Clone, Debug)] +pub struct C22(u8); + +impl C22 { + /// Basic mode control. + pub const BMCR: Self = C22(0x00); + /// Basic mode status. + pub const BMSR: Self = C22(0x01); + /// PHY identifier 1. + pub const PHYSID1: Self = C22(0x02); + /// PHY identifier 2. + pub const PHYSID2: Self = C22(0x03); + /// Auto-negotiation advertisement. + pub const ADVERTISE: Self = C22(0x04); + /// Auto-negotiation link partner base page ability. + pub const LPA: Self = C22(0x05); + /// Auto-negotiation expansion. + pub const EXPANSION: Self = C22(0x06); + /// Auto-negotiation next page transmit. + pub const NEXT_PAGE_TRANSMIT: Self = C22(0x07); + /// Auto-negotiation link partner received next page. + pub const LP_RECEIVED_NEXT_PAGE: Self = C22(0x08); + /// Master-slave control. + pub const MASTER_SLAVE_CONTROL: Self = C22(0x09); + /// Master-slave status. + pub const MASTER_SLAVE_STATUS: Self = C22(0x0a); + /// PSE Control. + pub const PSE_CONTROL: Self = C22(0x0b); + /// PSE Status. + pub const PSE_STATUS: Self = C22(0x0c); + /// MMD Register control. + pub const MMD_CONTROL: Self = C22(0x0d); + /// MMD Register address data. + pub const MMD_DATA: Self = C22(0x0e); + /// Extended status. + pub const EXTENDED_STATUS: Self = C22(0x0f); + + /// Creates a new instance of `C22` with a vendor specific register. + pub const fn vendor_specific() -> Self { + build_assert!( + N > 0x0f && N < 0x20, + "Vendor-specific register address must be between 16 and 31" + ); + C22(N) + } +} + +impl private::Sealed for C22 {} + +impl Register for C22 { + fn read(&self, dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer + // `phydev`. + let ret = unsafe { + bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, self.0.into()) + }; + to_result(ret)?; + Ok(ret as u16) + } + + fn write(&self, dev: &mut Device, val: u16) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer + // `phydev`. + to_result(unsafe { + bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, self.0.into(), val) + }) + } + + fn read_status(dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_read_status(phydev) }; + to_result(ret)?; + Ok(ret as u16) + } +} + +/// A single MDIO clause 45 register device and address. +#[derive(Copy, Clone, Debug)] +pub struct Mmd(u8); + +impl Mmd { + /// Physical Medium Attachment/Dependent. + pub const PMAPMD: Self = Mmd(uapi::MDIO_MMD_PMAPMD as u8); + /// WAN interface sublayer. + pub const WIS: Self = Mmd(uapi::MDIO_MMD_WIS as u8); + /// Physical coding sublayer. + pub const PCS: Self = Mmd(uapi::MDIO_MMD_PCS as u8); + /// PHY Extender sublayer. + pub const PHYXS: Self = Mmd(uapi::MDIO_MMD_PHYXS as u8); + /// DTE Extender sublayer. + pub const DTEXS: Self = Mmd(uapi::MDIO_MMD_DTEXS as u8); + /// Transmission convergence. + pub const TC: Self = Mmd(uapi::MDIO_MMD_TC as u8); + /// Auto negotiation. + pub const AN: Self = Mmd(uapi::MDIO_MMD_AN as u8); + /// Separated PMA (1). + pub const SEPARATED_PMA1: Self = Mmd(8); + /// Separated PMA (2). + pub const SEPARATED_PMA2: Self = Mmd(9); + /// Separated PMA (3). + pub const SEPARATED_PMA3: Self = Mmd(10); + /// Separated PMA (4). + pub const SEPARATED_PMA4: Self = Mmd(11); + /// OFDM PMA/PMD. + pub const OFDM_PMAPMD: Self = Mmd(12); + /// Power unit. + pub const POWER_UNIT: Self = Mmd(13); + /// Clause 22 extension. + pub const C22_EXT: Self = Mmd(uapi::MDIO_MMD_C22EXT as u8); + /// Vendor specific 1. + pub const VEND1: Self = Mmd(uapi::MDIO_MMD_VEND1 as u8); + /// Vendor specific 2. + pub const VEND2: Self = Mmd(uapi::MDIO_MMD_VEND2 as u8); +} + +/// A single MDIO clause 45 register device and address. +/// +/// Clause 45 uses a 5-bit device address to access a specific MMD within +/// a port, then a 16-bit register address to access a location within +/// that device. `C45` represents this by storing a [`Mmd`] and +/// a register number. +pub struct C45 { + devad: Mmd, + regnum: u16, +} + +impl C45 { + /// Creates a new instance of `C45`. + pub fn new(devad: Mmd, regnum: u16) -> Self { + Self { devad, regnum } + } +} + +impl private::Sealed for C45 {} + +impl Register for C45 { + fn read(&self, dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call. + let ret = + unsafe { bindings::phy_read_mmd(phydev, self.devad.0.into(), self.regnum.into()) }; + to_result(ret)?; + Ok(ret as u16) + } + + fn write(&self, dev: &mut Device, val: u16) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_write_mmd(phydev, self.devad.0.into(), self.regnum.into(), val) + }) + } + + fn read_status(dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_c45_read_status(phydev) }; + to_result(ret)?; + Ok(ret as u16) + } +} diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs new file mode 100644 index 000000000..de589792a --- /dev/null +++ b/rust/kernel/num.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Additional numerical features for the kernel. + +use core::ops; + +pub mod bounded; +pub mod casts; + +pub use bounded::*; + +/// Designates unsigned primitive types. +pub enum Unsigned {} + +/// Designates signed primitive types. +pub enum Signed {} + +mod private { + pub trait Sealed {} +} + +/// Describes core properties of integer types. +pub trait Integer: + private::Sealed + + Sized + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + ops::Add + + ops::AddAssign + + ops::Sub + + ops::SubAssign + + ops::Mul + + ops::MulAssign + + ops::Div + + ops::DivAssign + + ops::Rem + + ops::RemAssign + + ops::BitAnd + + ops::BitAndAssign + + ops::BitOr + + ops::BitOrAssign + + ops::BitXor + + ops::BitXorAssign + + ops::Shl + + ops::ShlAssign + + ops::Shr + + ops::ShrAssign + + ops::Not +{ + /// Whether this type is [`Signed`] or [`Unsigned`]. + type Signedness; + + /// Number of bits used for value representation. + const BITS: u32; +} + +macro_rules! impl_integer { + ($($type:ty: $signedness:ty), *) => { + $( + impl private::Sealed for $type {} + + impl Integer for $type { + type Signedness = $signedness; + + const BITS: u32 = <$type>::BITS; + } + )* + }; +} + +impl_integer!( + u8: Unsigned, + u16: Unsigned, + u32: Unsigned, + u64: Unsigned, + u128: Unsigned, + usize: Unsigned, + i8: Signed, + i16: Signed, + i32: Signed, + i64: Signed, + i128: Signed, + isize: Signed +); diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs new file mode 100644 index 000000000..2a2b0a4bc --- /dev/null +++ b/rust/kernel/num/bounded.rs @@ -0,0 +1,1165 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Bounded`], a wrapper around integer types limiting the number of bits +//! usable for value representation. + +use core::{ + cmp, + fmt, + ops::{ + self, + Deref, // + }, //, +}; + +use kernel::{ + num::{ + Integer, + Unsigned, // + }, + prelude::*, // +}; + +/// Evaluates to `true` if `$value` can be represented using at most `$n` bits in a `$type`. +/// +/// `expr` must be of type `type`, or the result will be incorrect. +/// +/// Can be used in const context. +macro_rules! fits_within { + ($value:expr, $type:ty, $n:expr) => {{ + let shift: u32 = <$type>::BITS - $n; + + // `value` fits within `$n` bits if shifting it left by the number of unused bits, then + // right by the same number, doesn't change it. + // + // This method has the benefit of working for both unsigned and signed values. + ($value << shift) >> shift == $value + }}; +} + +/// Returns `true` if `value` can be represented with at most `N` bits in a `T`. +#[inline(always)] +fn fits_within(value: T, num_bits: u32) -> bool { + fits_within!(value, T, num_bits) +} + +/// An integer value that requires only the `N` least significant bits of the wrapped type to be +/// encoded. +/// +/// This limits the number of usable bits in the wrapped integer type, and thus the stored value to +/// a narrower range, which provides guarantees that can be useful when working within e.g. +/// bitfields. +/// +/// # Invariants +/// +/// - `N` is greater than `0`. +/// - `N` is less than or equal to `T::BITS`. +/// - Stored values can be represented with at most `N` bits. +/// +/// # Examples +/// +/// The preferred way to create values is through constants and the [`Bounded::new`] family of +/// constructors, as they trigger a build error if the type invariants cannot be upheld. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // An unsigned 8-bit integer, of which only the 4 LSBs are used. +/// // The value `15` is statically validated to fit that constraint at build time. +/// let v = Bounded::::new::<15>(); +/// assert_eq!(v.get(), 15); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-8>(); +/// assert_eq!(v.get(), -8); +/// +/// // This doesn't build: a `u8` is smaller than the requested 9 bits. +/// // let _ = Bounded::::new::<10>(); +/// +/// // This also doesn't build: the requested value doesn't fit within 4 signed bits. +/// // let _ = Bounded::::new::<8>(); +/// ``` +/// +/// Values can also be validated at runtime with [`Bounded::try_new`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This succeeds because `15` can be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(15).is_some()); +/// +/// // This fails because `16` cannot be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(16).is_none()); +/// ``` +/// +/// Non-constant expressions can be validated at build-time thanks to compiler optimizations. This +/// should be used with caution, on simple expressions only. +/// +/// ``` +/// use kernel::num::Bounded; +/// # fn some_number() -> u32 { 0xffffffff } +/// +/// // Here the compiler can infer from the mask that the type invariants are not violated, even +/// // though the value returned by `some_number` is not statically known. +/// let v = Bounded::::from_expr(some_number() & 0xf); +/// ``` +/// +/// Comparison and arithmetic operations are supported on [`Bounded`]s with a compatible backing +/// type, regardless of their number of valid bits. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v1 = Bounded::::new::<4>(); +/// let v2 = Bounded::::new::<15>(); +/// +/// assert!(v1 != v2); +/// assert!(v1 < v2); +/// assert_eq!(v1 + v2, 19); +/// assert_eq!(v2 % v1, 3); +/// ``` +/// +/// These operations are also supported between a [`Bounded`] and its backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<15>(); +/// +/// assert!(v == 15); +/// assert!(v > 12); +/// assert_eq!(v + 5, 20); +/// assert_eq!(v / 3, 5); +/// ``` +/// +/// A change of backing types is possible using [`Bounded::cast`], and the number of valid bits can +/// be extended or reduced with [`Bounded::extend`] and [`Bounded::try_shrink`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<127>(); +/// +/// // Changes backing type from `u32` to `u16`. +/// let _: Bounded = v.cast(); +/// +/// // This does not build, as `u8` is smaller than 12 bits. +/// // let _: Bounded = v.cast(); +/// +/// // We can safely extend the number of bits... +/// let _ = v.extend::<15>(); +/// +/// // ... to the limits of the backing type. This doesn't build as a `u32` cannot contain 33 bits. +/// // let _ = v.extend::<33>(); +/// +/// // Reducing the number of bits is validated at runtime. This works because `127` can be +/// // represented with 8 bits. +/// assert!(v.try_shrink::<8>().is_some()); +/// +/// // ... but not with 6, so this fails. +/// assert!(v.try_shrink::<6>().is_none()); +/// ``` +/// +/// Infallible conversions from a primitive integer to a large-enough [`Bounded`] are supported. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This unsigned `Bounded` has 8 bits, so it can represent any `u8`. +/// let v = Bounded::::from(128u8); +/// assert_eq!(v.get(), 128); +/// +/// // This signed `Bounded` has 8 bits, so it can represent any `i8`. +/// let v = Bounded::::from(-128i8); +/// assert_eq!(v.get(), -128); +/// +/// // This doesn't build, as this 6-bit `Bounded` does not have enough capacity to represent a +/// // `u8` (regardless of the passed value). +/// // let _ = Bounded::::from(10u8); +/// +/// // Booleans can be converted into unsigned `Bounded`s. +/// +/// let v = Bounded::::from(false); +/// assert_eq!(v.get(), 0); +/// +/// let v = Bounded::::from(true); +/// assert_eq!(v.get(), 1); +/// +/// // This does not build because `i8` is signed. +/// // let _ = Bounded::::from(true); +/// ``` +/// +/// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and +/// dependent on the number of bits used for value representation, not on the backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // Even though its backing type is `u32`, this `Bounded` only uses 6 bits and thus can safely +/// // be converted to a `u8`. +/// let v = Bounded::::new::<63>(); +/// assert_eq!(u8::from(v), 63); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-128>(); +/// assert_eq!(i8::from(v), -128); +/// +/// // This however does not build, as 10 bits won't fit into a `u8` (regardless of the actually +/// // contained value). +/// let _v = Bounded::::new::<10>(); +/// // assert_eq!(u8::from(_v), 10); +/// +/// // Unsigned single-bit `Bounded`s can be converted into a boolean. +/// let v = Bounded::::new::<1>(); +/// assert_eq!(bool::from(v), true); +/// +/// let v = Bounded::::new::<0>(); +/// assert_eq!(bool::from(v), false); +/// +/// // This does not build because `i8` is signed. +/// // let v = Bounded::::new::<-1>(); +/// // let _ = bool::from(v); +/// ``` +/// +/// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the +/// [`TryIntoBounded`] trait. +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fit into 6 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v, None); +/// ``` +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Default, Hash)] +pub struct Bounded(T); + +/// Validating the value as a const expression cannot be done as a regular method, as the +/// arithmetic operations we rely on to check the bounds are not const. Thus, implement +/// [`Bounded::new`] using a macro. +macro_rules! impl_const_new { + ($($type:ty)*) => { + $( + impl Bounded<$type, N> { + /// Creates a [`Bounded`] for the constant `VALUE`. + /// + /// Fails at build time if `VALUE` cannot be represented with `N` bits. + /// + /// This method should be preferred to [`Self::from_expr`] whenever possible. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + #[doc = ::core::concat!( + "let v = Bounded::<", + ::core::stringify!($type), + ", 4>::new::<7>();")] + /// assert_eq!(v.get(), 7); + /// ``` + pub const fn new() -> Self { + // Statically assert that `VALUE` fits within the set number of bits. + const_assert!(fits_within!(VALUE, $type, N)); + + // SAFETY: `fits_within` confirmed that `VALUE` can be represented within + // `N` bits. + unsafe { Self::__new(VALUE) } + } + } + )* + }; +} + +impl_const_new!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +impl Bounded +where + T: Integer, +{ + /// Private constructor enforcing the type invariants. + /// + /// All instances of [`Bounded`] must be created through this method as it enforces most of the + /// type invariants. + /// + /// # Safety + /// + /// The caller must ensure that `value` can be represented within `N` bits. + const unsafe fn __new(value: T) -> Self { + // Enforce the type invariants. + // `N` cannot be zero. + const_assert!(N != 0); + // The backing type is at least as large as `N` bits. + const_assert!(N <= T::BITS); + + // INVARIANT: The caller ensures `value` fits within `N` bits. + Self(value) + } + + /// Attempts to turn `value` into a `Bounded` using `N` bits. + /// + /// Returns [`None`] if `value` doesn't fit within `N` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::try_new(1); + /// assert_eq!(v.as_deref().copied(), Some(1)); + /// + /// let v = Bounded::::try_new(-2); + /// assert_eq!(v.as_deref().copied(), Some(-2)); + /// + /// // `0x1ff` doesn't fit into 8 unsigned bits. + /// let v = Bounded::::try_new(0x1ff); + /// assert_eq!(v, None); + /// + /// // The range of values representable with 4 bits is `[-8..=7]`. The following tests these + /// // limits. + /// let v = Bounded::::try_new(-8); + /// assert_eq!(v.map(Bounded::get), Some(-8)); + /// let v = Bounded::::try_new(-9); + /// assert_eq!(v, None); + /// let v = Bounded::::try_new(7); + /// assert_eq!(v.map(Bounded::get), Some(7)); + /// let v = Bounded::::try_new(8); + /// assert_eq!(v, None); + /// ``` + pub fn try_new(value: T) -> Option { + fits_within(value, N).then(|| { + // SAFETY: `fits_within` confirmed that `value` can be represented within `N` bits. + unsafe { Self::__new(value) } + }) + } + + /// Checks that `expr` is valid for this type at compile-time and build a new value. + /// + /// This relies on [`build_assert!`] and guaranteed optimization to perform validation at + /// compile-time. If `expr` cannot be proved to be within the requested bounds at compile-time, + /// use the fallible [`Self::try_new`] instead. + /// + /// Limit this to simple, easily provable expressions, and prefer one of the [`Self::new`] + /// constructors whenever possible as they statically validate the value instead of relying on + /// compiler optimizations. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// # fn some_number() -> u32 { 0xffffffff } + /// + /// // Some undefined number. + /// let v: u32 = some_number(); + /// + /// // Triggers a build error as `v` cannot be asserted to fit within 4 bits... + /// // let _ = Bounded::::from_expr(v); + /// + /// // ... but this works as the compiler can assert the range from the mask. + /// let _ = Bounded::::from_expr(v & 0xf); + /// + /// // These expressions are simple enough to be proven correct, but since they are static the + /// // `new` constructor should be preferred. + /// assert_eq!(Bounded::::from_expr(1).get(), 1); + /// assert_eq!(Bounded::::from_expr(0xff).get(), 0xff); + /// ``` + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub fn from_expr(expr: T) -> Self { + crate::build_assert::build_assert!( + fits_within(expr, N), + "Requested value larger than maximal representable value." + ); + + // SAFETY: `fits_within` confirmed that `expr` can be represented within `N` bits. + unsafe { Self::__new(expr) } + } + + /// Returns the wrapped value as the backing type. + /// + /// This is similar to the [`Deref`] implementation, but doesn't enforce the size invariant of + /// the [`Bounded`], which might produce slightly less optimal code. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// assert_eq!(v.get(), 7u32); + /// ``` + pub const fn get(self) -> T { + self.0 + } + + /// Increases the number of bits usable for `self`. + /// + /// This operation cannot fail. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// let larger_v = v.extend::<12>(); + /// // The contained values are equal even though `larger_v` has a bigger capacity. + /// assert_eq!(larger_v, v); + /// ``` + pub const fn extend(self) -> Bounded { + const_assert!( + M >= N, + "Requested number of bits is less than the current representation." + ); + + // SAFETY: The value did fit within `N` bits, so it will all the more fit within + // the larger `M` bits. + unsafe { Bounded::__new(self.0) } + } + + /// Attempts to shrink the number of bits usable for `self`. + /// + /// Returns [`None`] if the value of `self` cannot be represented within `M` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// + /// // `7` can be represented using 3 unsigned bits... + /// let smaller_v = v.try_shrink::<3>(); + /// assert_eq!(smaller_v.as_deref().copied(), Some(7)); + /// + /// // ... but doesn't fit within `2` bits. + /// assert_eq!(v.try_shrink::<2>(), None); + /// ``` + pub fn try_shrink(self) -> Option> { + Bounded::::try_new(self.get()) + } + + /// Casts `self` into a [`Bounded`] backed by a different storage type, but using the same + /// number of valid bits. + /// + /// Both `T` and `U` must be of same signedness, and `U` must be at least as large as + /// `N` bits, or a build error will occur. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<127>(); + /// + /// let u16_v: Bounded = v.cast(); + /// assert_eq!(u16_v.get(), 127); + /// + /// // This won't build: a `u8` is smaller than the required 12 bits. + /// // let _: Bounded = v.cast(); + /// ``` + pub fn cast(self) -> Bounded + where + U: TryFrom + Integer, + T: Integer, + U: Integer, + { + // SAFETY: The converted value is represented using `N` bits, `U` can contain `N` bits, and + // `U` and `T` have the same sign, hence this conversion cannot fail. + let value = unsafe { U::try_from(self.get()).unwrap_unchecked() }; + + // SAFETY: Although the backing type has changed, the value is still represented within + // `N` bits, and with the same signedness. + unsafe { Bounded::__new(value) } + } + + /// Right-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N - SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff00>(); + /// let v_shifted: Bounded:: = v.shr::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff); + /// ``` + pub fn shr(self) -> Bounded { + const_assert!(SHIFT < T::BITS); + const_assert!(RES + SHIFT >= N); + + // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. + unsafe { Bounded::__new(self.0 >> SHIFT) } + } + + /// Right-shifts `self` by `SHIFT` if that loses no set bits, and returns the result as a + /// `Bounded<_, RES>`, where `RES >= N - SHIFT`. + /// + /// Returns [`None`] if any of the `SHIFT` least significant bits of `self` is set. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff00>(); + /// let v_shifted: Option> = v.shr_exact::<8, _>(); + /// + /// assert_eq!(v_shifted.map(|v| v.get()), Some(0xff)); + /// + /// // A set bit would be shifted out. + /// let v = Bounded::::new::<0xff01>(); + /// let v_shifted: Option> = v.shr_exact::<8, _>(); + /// + /// assert!(v_shifted.is_none()); + /// ``` + #[inline] + pub fn shr_exact(self) -> Option> { + let shifted = self.shr::(); + if shifted.get() << SHIFT == self.0 { + Some(shifted) + } else { + None + } + } + + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N + SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff>(); + /// let v_shifted: Bounded:: = v.shl::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff00); + /// ``` + pub fn shl(self) -> Bounded { + const_assert!(RES >= N + SHIFT); + + // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. + unsafe { Bounded::__new(self.0 << SHIFT) } + } +} + +impl Deref for Bounded +where + T: Integer, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + // Enforce the invariant to inform the compiler of the bounds of the value. + if !fits_within(self.0, N) { + // SAFETY: Per the `Bounded` invariants, `fits_within` can never return `false` on the + // value of a valid instance. + unsafe { core::hint::unreachable_unchecked() } + } + + &self.0 + } +} + +/// Trait similar to [`TryInto`] but for [`Bounded`], to avoid conflicting implementations. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fit into 6 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v, None); +/// ``` +pub trait TryIntoBounded { + /// Attempts to convert `self` into a [`Bounded`] using `N` bits. + /// + /// Returns [`None`] if `self` does not fit into the target type. + fn try_into_bounded(self) -> Option>; +} + +/// Any integer value can be attempted to be converted into a [`Bounded`] of any size. +impl TryIntoBounded for U +where + T: Integer, + U: TryInto, +{ + fn try_into_bounded(self) -> Option> { + self.try_into().ok().and_then(Bounded::try_new) + } +} + +// Comparisons between `Bounded`s. + +impl PartialEq> for Bounded +where + T: Integer, + U: Integer, + T: PartialEq, +{ + fn eq(&self, other: &Bounded) -> bool { + self.get() == other.get() + } +} + +impl Eq for Bounded where T: Integer {} + +impl PartialOrd> for Bounded +where + T: Integer, + U: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &Bounded) -> Option { + self.get().partial_cmp(&other.get()) + } +} + +impl Ord for Bounded +where + T: Integer, + T: Ord, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.get().cmp(&other.get()) + } +} + +// Comparisons between a `Bounded` and its backing type. + +impl PartialEq for Bounded +where + T: Integer, + T: PartialEq, +{ + fn eq(&self, other: &T) -> bool { + self.get() == *other + } +} + +impl PartialOrd for Bounded +where + T: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &T) -> Option { + self.get().partial_cmp(other) + } +} + +// Implementations of `core::ops` for two `Bounded` with the same backing type. + +impl ops::Add> for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: Bounded) -> Self::Output { + self.get() + rhs.get() + } +} + +impl ops::BitAnd> for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: Bounded) -> Self::Output { + self.get() & rhs.get() + } +} + +impl ops::BitOr> for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: Bounded) -> Self::Output { + self.get() | rhs.get() + } +} + +impl ops::BitXor> for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: Bounded) -> Self::Output { + self.get() ^ rhs.get() + } +} + +impl ops::Div> for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: Bounded) -> Self::Output { + self.get() / rhs.get() + } +} + +impl ops::Mul> for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: Bounded) -> Self::Output { + self.get() * rhs.get() + } +} + +impl ops::Rem> for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: Bounded) -> Self::Output { + self.get() % rhs.get() + } +} + +impl ops::Sub> for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: Bounded) -> Self::Output { + self.get() - rhs.get() + } +} + +// Implementations of `core::ops` between a `Bounded` and its backing type. + +impl ops::Add for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: T) -> Self::Output { + self.get() + rhs + } +} + +impl ops::BitAnd for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: T) -> Self::Output { + self.get() & rhs + } +} + +impl ops::BitOr for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: T) -> Self::Output { + self.get() | rhs + } +} + +impl ops::BitXor for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: T) -> Self::Output { + self.get() ^ rhs + } +} + +impl ops::Div for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: T) -> Self::Output { + self.get() / rhs + } +} + +impl ops::Mul for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: T) -> Self::Output { + self.get() * rhs + } +} + +impl ops::Neg for Bounded +where + T: Integer, + T: ops::Neg, +{ + type Output = T; + + fn neg(self) -> Self::Output { + -self.get() + } +} + +impl ops::Not for Bounded +where + T: Integer, + T: ops::Not, +{ + type Output = T; + + fn not(self) -> Self::Output { + !self.get() + } +} + +impl ops::Rem for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: T) -> Self::Output { + self.get() % rhs + } +} + +impl ops::Sub for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: T) -> Self::Output { + self.get() - rhs + } +} + +// Proxy implementations of `core::fmt`. + +impl fmt::Display for Bounded +where + T: Integer, + T: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Binary for Bounded +where + T: Integer, + T: fmt::Binary, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerExp for Bounded +where + T: Integer, + T: fmt::LowerExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerHex for Bounded +where + T: Integer, + T: fmt::LowerHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Octal for Bounded +where + T: Integer, + T: fmt::Octal, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperExp for Bounded +where + T: Integer, + T: fmt::UpperExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperHex for Bounded +where + T: Integer, + T: fmt::UpperHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +/// Implements `$trait` for all [`Bounded`] types represented using `$num_bits`. +/// +/// This is used to declare size properties as traits that we can constrain against in impl blocks. +macro_rules! impl_size_rule { + ($trait:ty, $($num_bits:literal)*) => { + $( + impl $trait for Bounded where T: Integer {} + )* + }; +} + +/// Local trait expressing the fact that a given [`Bounded`] has at least `N` bits used for value +/// representation. +trait AtLeastXBits {} + +/// Implementations for infallibly converting a primitive type into a [`Bounded`] that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod atleast_impls { + use super::*; + + // Number of bits at least as large as 64. + impl_size_rule!(AtLeastXBits<64>, 64); + + // Anything 64 bits or more is also larger than 32. + impl AtLeastXBits<32> for T where T: AtLeastXBits<64> {} + // Other numbers of bits at least as large as 32. + impl_size_rule!(AtLeastXBits<32>, + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + ); + + // Anything 32 bits or more is also larger than 16. + impl AtLeastXBits<16> for T where T: AtLeastXBits<32> {} + // Other numbers of bits at least as large as 16. + impl_size_rule!(AtLeastXBits<16>, + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + ); + + // Anything 16 bits or more is also larger than 8. + impl AtLeastXBits<8> for T where T: AtLeastXBits<16> {} + // Other numbers of bits at least as large as 8. + impl_size_rule!(AtLeastXBits<8>, 8 9 10 11 12 13 14 15); +} + +/// Generates `From` implementations from a primitive type into a [`Bounded`] with +/// enough bits to store any value of that type. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of [`AtLeastXBits`]'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_from_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`", + ::core::stringify!($type), + "`] into a [`Bounded`] of same signedness with enough bits to store it.")] + impl From<$type> for Bounded + where + $type: Integer, + T: Integer::Signedness> + From<$type>, + Self: AtLeastXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: $type) -> Self { + // SAFETY: The trait bound on `Self` guarantees that `N` bits is + // enough to hold any value of the source type. + unsafe { Self::__new(T::from(value)) } + } + } + )* + } +} + +impl_from_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +/// Local trait expressing the fact that a given [`Bounded`] fits into a primitive type of `N` bits, +/// provided they have the same signedness. +trait FitsInXBits {} + +/// Implementations for infallibly converting a [`Bounded`] into a primitive type that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod fits_impls { + use super::*; + + // Number of bits that fit into a 8-bits primitive. + impl_size_rule!(FitsInXBits<8>, 1 2 3 4 5 6 7 8); + + // Anything that fits into 8 bits also fits into 16. + impl FitsInXBits<16> for T where T: FitsInXBits<8> {} + // Other number of bits that fit into a 16-bits primitive. + impl_size_rule!(FitsInXBits<16>, 9 10 11 12 13 14 15 16); + + // Anything that fits into 16 bits also fits into 32. + impl FitsInXBits<32> for T where T: FitsInXBits<16> {} + // Other number of bits that fit into a 32-bits primitive. + impl_size_rule!(FitsInXBits<32>, + 17 18 19 20 21 22 23 24 + 25 26 27 28 29 30 31 32 + ); + + // Anything that fits into 32 bits also fits into 64. + impl FitsInXBits<64> for T where T: FitsInXBits<32> {} + // Other number of bits that fit into a 64-bits primitive. + impl_size_rule!(FitsInXBits<64>, + 33 34 35 36 37 38 39 40 + 41 42 43 44 45 46 47 48 + 49 50 51 52 53 54 55 56 + 57 58 59 60 61 62 63 64 + ); +} + +/// Generates [`From`] implementations from a [`Bounded`] into a primitive type that is +/// guaranteed to contain it. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of `AtLeastXBits`'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_into_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`Bounded`] with no more bits than a [`", + ::core::stringify!($type), + "`] and of same signedness into [`", + ::core::stringify!($type), + "`]")] + impl From> for $type + where + $type: Integer + TryFrom, + T: Integer::Signedness>, + Bounded: FitsInXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: Bounded) -> $type { + // SAFETY: The trait bound on `Bounded` ensures that any value it holds (which + // is constrained to `N` bits) can fit into the destination type, so this + // conversion cannot fail. + unsafe { <$type>::try_from(value.get()).unwrap_unchecked() } + } + } + )* + } +} + +impl_into_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +// Unsigned single-bit `Bounded`s can be converted to a boolean. + +impl From> for bool +where + T: Integer + Zeroable, +{ + fn from(value: Bounded) -> Self { + value.get() != Zeroable::zeroed() + } +} + +// Booleans can be converted to unsigned `Bounded`s. + +impl From for Bounded +where + T: Integer + From, +{ + fn from(value: bool) -> Self { + // SAFETY: A boolean is represented by `0` or `1`, so it fits within any valid unsigned + // `Bounded` width. + unsafe { Self::__new(T::from(value)) } + } +} + +impl Bounded +where + T: Integer + Zeroable, +{ + /// Converts this [`Bounded`] into a [`bool`]. + /// + /// This is a shorter way of writing `bool::from(self)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// assert_eq!(Bounded::::new::<0>().into_bool(), false); + /// assert_eq!(Bounded::::new::<1>().into_bool(), true); + /// ``` + pub fn into_bool(self) -> bool { + self.into() + } +} diff --git a/rust/kernel/num/casts.rs b/rust/kernel/num/casts.rs new file mode 100644 index 000000000..7e6c7dec7 --- /dev/null +++ b/rust/kernel/num/casts.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helpers for performing lossless integer casts. +//! +//! The `as` keyword can be used to perform casts between integer types, but it unfortunately makes +//! no distinction between casts that are lossless, and casts from a larger type into a smaller one +//! that might silently strip data away. Thus, its use in the kernel is discouraged in favor of +//! [`From`] implementations. +//! +//! Conversely, there are casts that are lossless depending on the build architecture (such as +//! casting [`usize`] to [`u64`] on 32 or 64 bit archs), but not supported by [`From`] +//! implementations in the standard library because they are not portable. It does however make +//! sense for the kernel to support these, if only for code that is architecture-specific. +//! +//! This module provides ways to perform such conversions safely: +//! +//! - A series of const functions (e.g. [`usize_as_u64`]) supporting safe conversions in const +//! context. Conversions supported by [`From`] implementations in the standard library are also +//! covered as the [`From`] trait cannot be used in const context. +//! - Two extension traits, [`FromSafeCast`] and [`IntoSafeCast`], providing conversion methods +//! similar to [`From`] and [`Into`] for conversions that are safe to perform in the kernel, but +//! not supported by the standard library. +//! - Another series of const functions (e.g. [`u64_into_u8`]) supporting the conversion of a const +//! value from a larger type into a smaller one, provided the value fits into the destination +//! type. This is useful if a constant is defined as a larger type, but needs to be used as a +//! smaller one. +//! - An [`arch`] sub-module, defining more conversion functions that are only guaranteed to be +//! lossless for a given pointer size. These can only be used in code that is specific to a +//! given pointer size. +//! +//! # Examples +//! +//! ``` +//! use kernel::num::casts::{self, FromSafeCast, IntoSafeCast}; +//! +//! // Conversion from const context. +//! const USIZED_CONST: usize = casts::u8_as_usize(255u8); +//! +//! // Non-const conversions. +//! let a = u64::from_safe_cast(4096usize); +//! let b: u64 = 4096usize.into_safe_cast(); +//! ``` + +use crate::prelude::*; + +/// Implements safe `as` conversion functions from a given type into a series of target types. +/// +/// These functions can be used in place of `as`, with the guarantee that they will be lossless. +macro_rules! impl_safe_as { + ($from:ty as { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Losslessly converts a [`", + ::core::stringify!($from), + "`] into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This conversion is allowed as it is always lossless. Prefer this over the `as` + /// keyword to ensure no lossy casts are performed. + /// + /// This is for use from a `const` context. For non `const` use, prefer the + /// [`FromSafeCast`] and [`IntoSafeCast`] traits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_as_", + ::core::stringify!($into), + "(1", + ::core::stringify!($from), + "), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _as_ $into>](value: $from) -> $into { + $crate::static_assert!(size_of::<$into>() >= size_of::<$from>()); + + value as $into + } + } + )* + }; +} + +// Valid `Into` transformations. +impl_safe_as!(u8 as { u16, u32, u64, usize }); +impl_safe_as!(u16 as { u32, u64, usize }); +impl_safe_as!(u32 as { u64 }); +// A `usize` fits into a `u64` on all supported platforms. +impl_safe_as!(usize as { u64 }); +// A `u32` fits into a `usize` on all supported platforms. +impl_safe_as!(u32 as { usize }); + +/// Extension trait providing guaranteed lossless cast to [`Self`] from `T`. +/// +/// The standard library's [`From`] implementations do not cover conversions that are not portable +/// or future-proof. For instance, even though it is safe today, [`From`] is not implemented +/// for [`u64`] because of the possibility of needing to support larger-than-64bit architectures in +/// the future. +/// +/// The workaround is to either deal with the error handling of [`TryFrom`] for an operation that +/// technically cannot fail, or to use the `as` keyword, which can silently strip data if the +/// destination type is smaller than the source. +/// +/// Both options are hardly acceptable for the kernel. It is also a much more architecture +/// dependent environment, supporting only 32 and 64 bit architectures, with some modules +/// explicitly depending on a specific bus width that could greatly benefit from infallible +/// conversion operations. +/// +/// Thus this extension trait that provides, for all architectures supported by the kernel, +/// conversion methods between types for which such a cast is lossless. +/// +/// In other words, this trait is implemented if, for all supported targets and with `t: T`, the +/// `t as Self` operation is completely lossless. +/// +/// Prefer this over the `as` keyword to guarantee that no lossy casts are performed. +/// +/// If you need to perform a conversion in `const` context, use [`u32_as_usize`], [`usize_as_u64`], +/// etc. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::FromSafeCast; +/// +/// assert_eq!(usize::from_safe_cast(0xf00u32), 0xf00usize); +/// ``` +pub trait FromSafeCast { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast(value: T) -> Self; +} + +// A `usize` fits into a `u64` on all supported platforms. +impl FromSafeCast for u64 { + #[inline] + fn from_safe_cast(value: usize) -> Self { + usize_as_u64(value) + } +} + +// A `u32` fits into a `usize` on all supported platforms. +impl FromSafeCast for usize { + #[inline] + fn from_safe_cast(value: u32) -> Self { + u32_as_usize(value) + } +} + +/// Counterpart to the [`FromSafeCast`] trait, i.e. this trait is to [`FromSafeCast`] what [`Into`] +/// is to [`From`]. +/// +/// See the documentation of [`FromSafeCast`] for the motivation. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::IntoSafeCast; +/// +/// assert_eq!(0xf00usize, 0xf00u32.into_safe_cast()); +/// ``` +pub trait IntoSafeCast { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast(self) -> T; +} + +/// Reverse operation for types implementing [`FromSafeCast`]. +impl IntoSafeCast for S +where + T: FromSafeCast, +{ + #[inline] + fn into_safe_cast(self) -> T { + T::from_safe_cast(self) + } +} + +/// Implements lossless conversion of a constant from a larger type into a smaller one. +macro_rules! impl_const_into { + ($from:ty => { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Performs a build-time safe conversion of a [`", + ::core::stringify!($from), + "`] constant value into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This checks at compile-time that the conversion is lossless, and triggers a build + /// error if it isn't. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + /// // Succeeds because the value of the source fits into the destination's type. + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_into_", + ::core::stringify!($into), + "::<1", + ::core::stringify!($from), + ">(), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _into_ $into>]() -> $into { + // Make sure that the target type is smaller than the source one. + $crate::static_assert!($from::BITS >= $into::BITS); + // CAST: we statically enforced above that `$from` is larger than `$into`, so the + // `as` conversion will be lossless. + $crate::const_assert!(N >= $into::MIN as $from && N <= $into::MAX as $from); + + N as $into + } + } + )* + }; +} + +impl_const_into!(usize => { u8, u16, u32 }); +impl_const_into!(u64 => { u8, u16, u32 }); +impl_const_into!(u32 => { u8, u16 }); +impl_const_into!(u16 => { u8 }); + +/// Conversions that are only lossless for the current architecture. +/// +/// # Portability +/// +/// Callers of this module become dependent on the setting of `CONFIG_64BIT`. Use with caution, and +/// never in code that is portable across pointer sizes. +pub mod arch { + /// Trait identical to [`FromSafeCast`](super::FromSafeCast), but for conversions that are not + /// available on all architectures. + pub trait FromSafeCastArch { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast_arch(value: T) -> Self; + } + + /// Trait identical to [`IntoSafeCast`](super::IntoSafeCast), but for conversions that are not + /// available on all architectures. + pub trait IntoSafeCastArch { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast_arch(self) -> T; + } + + /// Reverse operation for types implementing [`FromSafeCastArch`]. + impl IntoSafeCastArch for S + where + T: FromSafeCastArch, + { + #[inline] + fn into_safe_cast_arch(self) -> T { + T::from_safe_cast_arch(self) + } + } + + /// A [`u64`] fits into a [`usize`] on 64-bit platforms. + #[cfg(CONFIG_64BIT)] + #[inline] + pub const fn u64_as_usize(value: u64) -> usize { + value as usize + } + + #[cfg(CONFIG_64BIT)] + impl FromSafeCastArch for usize { + #[inline] + fn from_safe_cast_arch(value: u64) -> Self { + u64_as_usize(value) + } + } + + /// A [`usize`] fits into a [`u32`] on 32-bit platforms. + #[cfg(not(CONFIG_64BIT))] + #[inline] + pub const fn usize_as_u32(value: usize) -> u32 { + value as u32 + } + + #[cfg(not(CONFIG_64BIT))] + impl FromSafeCastArch for u32 { + #[inline] + fn from_safe_cast_arch(value: usize) -> Self { + usize_as_u32(value) + } + } +} diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 000000000..d0318f62a --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Device Tree / Open Firmware abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for OF drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// An open firmware device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::of_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` and +// does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); +} + +impl DeviceId { + /// Create a new device id from an OF 'compatible' string. + pub const fn new(compatible: &'static CStr) -> Self { + let src = compatible.to_bytes_with_nul(); + // Replace with `bindings::of_device_id::default()` once stabilized for `const`. + // SAFETY: FFI type is valid to be zero-initialized. + let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() }; + + // TODO: Use `copy_from_slice` once stabilized for `const`. + let mut i = 0; + while i < src.len() { + of.compatible[i] = src[i]; + i += 1; + } + + Self(of) + } +} + +/// Create an OF `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! of_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("of", $crate::of::DeviceId, $($tt)*); + }; +} diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs new file mode 100644 index 000000000..62e446761 --- /dev/null +++ b/rust/kernel/opp.rs @@ -0,0 +1,1148 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Operating performance points. +//! +//! This module provides rust abstractions for interacting with the OPP subsystem. +//! +//! C header: [`include/linux/pm_opp.h`](srctree/include/linux/pm_opp.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + cpumask::{Cpumask, CpumaskVar}, + device::Device, + error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + ffi::{c_char, c_ulong}, + prelude::*, + str::CString, + sync::aref::{ARef, AlwaysRefCounted}, + types::Opaque, +}; + +#[cfg(CONFIG_CPU_FREQ)] +/// Frequency table implementation. +mod freq { + use super::*; + use crate::cpufreq; + use core::ops::Deref; + + /// OPP frequency table. + /// + /// A [`cpufreq::Table`] created from [`Table`]. + pub struct FreqTable { + dev: ARef, + ptr: *mut bindings::cpufreq_frequency_table, + } + + impl FreqTable { + /// Creates a new instance of [`FreqTable`] from [`Table`]. + pub(crate) fn new(table: &Table) -> Result { + let mut ptr: *mut bindings::cpufreq_frequency_table = ptr::null_mut(); + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_init_cpufreq_table(table.dev.as_raw(), &mut ptr) + })?; + + Ok(Self { + dev: table.dev.clone(), + ptr, + }) + } + + /// Returns a reference to the underlying [`cpufreq::Table`]. + #[inline] + fn table(&self) -> &cpufreq::Table { + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { cpufreq::Table::from_raw(self.ptr) } + } + } + + impl Deref for FreqTable { + type Target = cpufreq::Table; + + #[inline] + fn deref(&self) -> &Self::Target { + self.table() + } + } + + impl Drop for FreqTable { + fn drop(&mut self) { + // SAFETY: The pointer was created via `dev_pm_opp_init_cpufreq_table`, and is only + // freed here. + unsafe { + bindings::dev_pm_opp_free_cpufreq_table(self.dev.as_raw(), &mut self.as_raw()) + }; + } + } +} + +#[cfg(CONFIG_CPU_FREQ)] +pub use freq::FreqTable; + +use core::{marker::PhantomData, ptr}; + +use macros::vtable; + +/// Creates a null-terminated slice of pointers to [`CString`]s. +fn to_c_str_array(names: &[CString]) -> Result> { + // Allocated a null-terminated vector of pointers. + let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?; + + for name in names.iter() { + list.push(name.as_char_ptr(), GFP_KERNEL)?; + } + + list.push(ptr::null(), GFP_KERNEL)?; + Ok(list) +} + +/// The voltage unit. +/// +/// Represents voltage in microvolts, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::opp::MicroVolt; +/// +/// let raw = 90500; +/// let volt = MicroVolt(raw); +/// +/// assert_eq!(usize::from(volt), raw); +/// assert_eq!(volt, MicroVolt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroVolt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(volt: MicroVolt) -> Self { + volt.0 + } +} + +/// The power unit. +/// +/// Represents power in microwatts, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::opp::MicroWatt; +/// +/// let raw = 1000000; +/// let power = MicroWatt(raw); +/// +/// assert_eq!(usize::from(power), raw); +/// assert_eq!(power, MicroWatt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroWatt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(power: MicroWatt) -> Self { + power.0 + } +} + +/// Handle for a dynamically created [`OPP`]. +/// +/// The associated [`OPP`] is automatically removed when the [`Token`] is dropped. +/// +/// # Examples +/// +/// The following example demonstrates how to create an [`OPP`] dynamically. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::sync::aref::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +pub struct Token { + dev: ARef, + freq: Hertz, +} + +impl Token { + /// Dynamically adds an [`OPP`] and returns a [`Token`] that removes it on drop. + fn new(dev: &ARef, mut data: Data) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_add_dynamic(dev.as_raw(), &mut data.0) })?; + Ok(Self { + dev: dev.clone(), + freq: data.freq(), + }) + } +} + +impl Drop for Token { + fn drop(&mut self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_remove(self.dev.as_raw(), self.freq.into()) }; + } +} + +/// OPP data. +/// +/// Rust abstraction for the C `struct dev_pm_opp_data`, used to define operating performance +/// points (OPPs) dynamically. +/// +/// # Examples +/// +/// The following example demonstrates how to create an [`OPP`] with [`Data`]. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::sync::aref::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +#[repr(transparent)] +pub struct Data(bindings::dev_pm_opp_data); + +impl Data { + /// Creates a new instance of [`Data`]. + /// + /// This can be used to define a dynamic OPP to be added to a device. + pub fn new(freq: Hertz, volt: MicroVolt, level: u32, turbo: bool) -> Self { + Self(bindings::dev_pm_opp_data { + turbo, + freq: freq.into(), + u_volt: volt.into(), + level, + }) + } + + /// Adds an [`OPP`] dynamically. + /// + /// Returns a [`Token`] that ensures the OPP is automatically removed + /// when it goes out of scope. + #[inline] + pub fn add_opp(self, dev: &ARef) -> Result { + Token::new(dev, self) + } + + /// Returns the frequency associated with this OPP data. + #[inline] + fn freq(&self) -> Hertz { + Hertz(self.0.freq) + } +} + +/// [`OPP`] search options. +/// +/// # Examples +/// +/// Defines how to search for an [`OPP`] in a [`Table`] relative to a frequency. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{OPP, SearchType, Table}; +/// use kernel::sync::aref::ARef; +/// +/// fn find_opp(table: &Table, freq: Hertz) -> Result> { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// pr_info!("OPP frequency is: {:?}\n", opp.freq(None)); +/// pr_info!("OPP voltage is: {:?}\n", opp.voltage()); +/// pr_info!("OPP level is: {}\n", opp.level()); +/// pr_info!("OPP power is: {:?}\n", opp.power()); +/// +/// Ok(opp) +/// } +/// ``` +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum SearchType { + /// Match the exact frequency. + Exact, + /// Find the highest frequency less than or equal to the given value. + Floor, + /// Find the lowest frequency greater than or equal to the given value. + Ceil, +} + +/// OPP configuration callbacks. +/// +/// Implement this trait to customize OPP clock and regulator setup for your device. +#[vtable] +pub trait ConfigOps { + /// This is typically used to scale clocks when transitioning between OPPs. + #[inline] + fn config_clks(_dev: &Device, _table: &Table, _opp: &OPP, _scaling_down: bool) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// This provides access to the old and new OPPs, allowing for safe regulator adjustments. + #[inline] + fn config_regulators( + _dev: &Device, + _opp_old: &OPP, + _opp_new: &OPP, + _data: *mut *mut bindings::regulator, + _count: u32, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// OPP configuration token. +/// +/// Returned by the OPP core when configuration is applied to a [`Device`]. The associated +/// configuration is automatically cleared when the token is dropped. +pub struct ConfigToken(i32); + +impl Drop for ConfigToken { + fn drop(&mut self) { + // SAFETY: This is the same token value returned by the C code via `dev_pm_opp_set_config`. + unsafe { bindings::dev_pm_opp_clear_config(self.0) }; + } +} + +/// OPP configurations. +/// +/// Rust abstraction for the C `struct dev_pm_opp_config`. +/// +/// # Examples +/// +/// The following example demonstrates how to set OPP property-name configuration for a [`Device`]. +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Config, ConfigOps, ConfigToken}; +/// use kernel::str::CString; +/// use kernel::sync::aref::ARef; +/// use kernel::macros::vtable; +/// +/// #[derive(Default)] +/// struct Driver; +/// +/// #[vtable] +/// impl ConfigOps for Driver {} +/// +/// fn configure(dev: &ARef) -> Result { +/// let name = CString::try_from_fmt(fmt!("slow"))?; +/// +/// // The OPP configuration is cleared once the [`ConfigToken`] goes out of scope. +/// Config::::new() +/// .set_prop_name(name)? +/// .set(dev) +/// } +/// ``` +#[derive(Default)] +pub struct Config +where + T: Default, +{ + clk_names: Option>, + prop_name: Option, + regulator_names: Option>, + supported_hw: Option>, + + // Tuple containing (required device, index) + required_dev: Option<(ARef, u32)>, + _data: PhantomData, +} + +impl Config { + /// Creates a new instance of [`Config`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Initializes clock names. + pub fn set_clk_names(mut self, names: KVec) -> Result { + if self.clk_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.clk_names = Some(names); + Ok(self) + } + + /// Initializes property name. + pub fn set_prop_name(mut self, name: CString) -> Result { + if self.prop_name.is_some() { + return Err(EBUSY); + } + + self.prop_name = Some(name); + Ok(self) + } + + /// Initializes regulator names. + pub fn set_regulator_names(mut self, names: KVec) -> Result { + if self.regulator_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.regulator_names = Some(names); + + Ok(self) + } + + /// Initializes required devices. + pub fn set_required_dev(mut self, dev: ARef, index: u32) -> Result { + if self.required_dev.is_some() { + return Err(EBUSY); + } + + self.required_dev = Some((dev, index)); + Ok(self) + } + + /// Initializes supported hardware. + pub fn set_supported_hw(mut self, hw: KVec) -> Result { + if self.supported_hw.is_some() { + return Err(EBUSY); + } + + if hw.is_empty() { + return Err(EINVAL); + } + + self.supported_hw = Some(hw); + Ok(self) + } + + /// Sets the configuration with the OPP core. + /// + /// The returned [`ConfigToken`] will remove the configuration when dropped. + pub fn set(self, dev: &Device) -> Result { + let clk_names = self.clk_names.as_deref().map(to_c_str_array).transpose()?; + let regulator_names = self + .regulator_names + .as_deref() + .map(to_c_str_array) + .transpose()?; + + let set_config = || { + let clk_names = clk_names.as_ref().map_or(ptr::null(), |c| c.as_ptr()); + let regulator_names = regulator_names.as_ref().map_or(ptr::null(), |c| c.as_ptr()); + + let prop_name = self + .prop_name + .as_ref() + .map_or(ptr::null(), |p| p.as_char_ptr()); + + let (supported_hw, supported_hw_count) = self + .supported_hw + .as_ref() + .map_or((ptr::null(), 0), |hw| (hw.as_ptr(), hw.len() as u32)); + + let (required_dev, required_dev_index) = self + .required_dev + .as_ref() + .map_or((ptr::null_mut(), 0), |(dev, idx)| (dev.as_raw(), *idx)); + + let mut config = bindings::dev_pm_opp_config { + clk_names, + config_clks: if T::HAS_CONFIG_CLKS { + Some(Self::config_clks) + } else { + None + }, + prop_name, + regulator_names, + config_regulators: if T::HAS_CONFIG_REGULATORS { + Some(Self::config_regulators) + } else { + None + }, + supported_hw, + supported_hw_count, + + required_dev, + required_dev_index, + }; + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The OPP core guarantees not to access fields of [`Config`] after this + // call and so we don't need to save a copy of them for future use. + let ret = unsafe { bindings::dev_pm_opp_set_config(dev.as_raw(), &mut config) }; + + to_result(ret).map(|()| ConfigToken(ret)) + }; + + // Ensure the closure does not accidentally drop owned data; if violated, the compiler + // produces E0525 with e.g.: + // + // ``` + // closure is `FnOnce` because it moves the variable `clk_names` out of its environment + // ``` + let _: &dyn Fn() -> _ = &set_config; + + set_config() + } + + /// Config's clk callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_clks( + dev: *mut bindings::device, + opp_table: *mut bindings::opp_table, + opp: *mut bindings::dev_pm_opp, + _data: *mut c_void, + scaling_down: bool, + ) -> c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_clks( + &dev, + // SAFETY: 'opp_table' is guaranteed by the C code to be valid. + &unsafe { Table::from_raw_table(opp_table, &dev) }, + // SAFETY: 'opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(opp)? }, + scaling_down, + ) + .map(|()| 0) + }) + } + + /// Config's regulator callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_regulators( + dev: *mut bindings::device, + old_opp: *mut bindings::dev_pm_opp, + new_opp: *mut bindings::dev_pm_opp, + regulators: *mut *mut bindings::regulator, + count: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_regulators( + &dev, + // SAFETY: 'old_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(old_opp)? }, + // SAFETY: 'new_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(new_opp)? }, + regulators, + count, + ) + .map(|()| 0) + }) + } +} + +/// A reference-counted OPP table. +/// +/// Rust abstraction for the C `struct opp_table`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`Table`]. +/// +/// Instances of this type are reference-counted. +/// +/// # Examples +/// +/// The following example demonstrates how to get OPP [`Table`] for a [`Cpumask`] and set its +/// frequency. +/// +/// ``` +/// # #![cfg(CONFIG_OF)] +/// use kernel::clk::Hertz; +/// use kernel::cpumask::Cpumask; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::Table; +/// use kernel::sync::aref::ARef; +/// +/// fn get_table(dev: &ARef, mask: &mut Cpumask, freq: Hertz) -> Result { +/// let mut opp_table = Table::from_of_cpumask(dev, mask)?; +/// +/// if opp_table.opp_count()? == 0 { +/// return Err(EINVAL); +/// } +/// +/// pr_info!("Max transition latency is: {} ns\n", opp_table.max_transition_latency_ns()); +/// pr_info!("Suspend frequency is: {:?}\n", opp_table.suspend_freq()); +/// +/// opp_table.set_rate(freq)?; +/// Ok(opp_table) +/// } +/// ``` +pub struct Table { + ptr: *mut bindings::opp_table, + dev: ARef, + #[allow(dead_code)] + em: bool, + #[allow(dead_code)] + of: bool, + cpus: Option, +} + +/// SAFETY: It is okay to send ownership of [`Table`] across thread boundaries. +unsafe impl Send for Table {} + +/// SAFETY: It is okay to access [`Table`] through shared references from other threads because +/// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for Table {} + +impl Table { + /// Creates a new reference-counted [`Table`] from a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid and non-null. + unsafe fn from_raw_table(ptr: *mut bindings::opp_table, dev: &ARef) -> Self { + // SAFETY: By the safety requirements, ptr is valid and its refcount will be incremented. + // + // INVARIANT: The reference-count is decremented when [`Table`] goes out of scope. + unsafe { bindings::dev_pm_opp_get_opp_table_ref(ptr) }; + + Self { + ptr, + dev: dev.clone(), + em: false, + of: false, + cpus: None, + } + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`]. + pub fn from_dev(dev: &Device) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + let ptr = from_err_ptr(unsafe { bindings::dev_pm_opp_get_opp_table(dev.as_raw()) })?; + + Ok(Self { + ptr, + dev: dev.into(), + em: false, + of: false, + cpus: None, + }) + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of(dev: &ARef, index: i32) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_add_table_indexed(dev.as_raw(), index) })?; + + // Get the newly created [`Table`]. + let mut table = Self::from_dev(dev)?; + table.of = true; + + Ok(table) + } + + /// Remove device tree based [`Table`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of(&self) { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. We took the reference from [`from_of`] earlier, it is safe to drop the + // same now. + unsafe { bindings::dev_pm_opp_of_remove_table(self.dev.as_raw()) }; + } + + /// Creates a new reference-counted [`Table`] instance for a [`Cpumask`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of_cpumask(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The cpumask is valid and the returned pointer will be owned by the [`Table`] + // instance. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_cpumask_add_table(cpumask.as_raw()) })?; + + // Fetch the newly created table. + let mut table = Self::from_dev(dev)?; + table.cpus = Some(CpumaskVar::try_clone(cpumask)?); + + Ok(table) + } + + /// Remove device tree based [`Table`] for a [`Cpumask`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of_cpumask(&self, cpumask: &Cpumask) { + // SAFETY: The cpumask is valid and we took the reference from [`from_of_cpumask`] earlier, + // it is safe to drop the same now. + unsafe { bindings::dev_pm_opp_of_cpumask_remove_table(cpumask.as_raw()) }; + } + + /// Returns the number of [`OPP`]s in the [`Table`]. + pub fn opp_count(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + let ret = unsafe { bindings::dev_pm_opp_get_opp_count(self.dev.as_raw()) }; + + to_result(ret).map(|()| ret as u32) + } + + /// Returns max clock latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_clock_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_clock_latency(self.dev.as_raw()) } + } + + /// Returns max volt latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_volt_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_volt_latency(self.dev.as_raw()) } + } + + /// Returns max transition latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_transition_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_transition_latency(self.dev.as_raw()) } + } + + /// Returns the suspend [`OPP`]'s frequency. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + Hertz(unsafe { bindings::dev_pm_opp_get_suspend_opp_freq(self.dev.as_raw()) }) + } + + /// Synchronizes regulators used by the [`Table`]. + #[inline] + pub fn sync_regulators(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_sync_regulators(self.dev.as_raw()) }) + } + + /// Gets sharing CPUs. + #[inline] + pub fn sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) }) + } + + /// Sets sharing CPUs. + pub fn set_sharing_cpus(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_set_sharing_cpus(self.dev.as_raw(), cpumask.as_raw()) + })?; + + if let Some(mask) = self.cpus.as_mut() { + // Update the cpumask as this will be used while removing the table. + cpumask.copy(mask); + } + + Ok(()) + } + + /// Gets sharing CPUs from device tree. + #[cfg(CONFIG_OF)] + #[inline] + pub fn of_sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) + }) + } + + /// Updates the voltage value for an [`OPP`]. + #[inline] + pub fn adjust_voltage( + &self, + freq: Hertz, + volt: MicroVolt, + volt_min: MicroVolt, + volt_max: MicroVolt, + ) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_adjust_voltage( + self.dev.as_raw(), + freq.into(), + volt.into(), + volt_min.into(), + volt_max.into(), + ) + }) + } + + /// Creates [`FreqTable`] from [`Table`]. + #[cfg(CONFIG_CPU_FREQ)] + #[inline] + pub fn cpufreq_table(&mut self) -> Result { + FreqTable::new(self) + } + + /// Configures device with [`OPP`] matching the frequency value. + #[inline] + pub fn set_rate(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_rate(self.dev.as_raw(), freq.into()) }) + } + + /// Configures device with [`OPP`]. + #[inline] + pub fn set_opp(&self, opp: &OPP) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_opp(self.dev.as_raw(), opp.as_raw()) }) + } + + /// Finds [`OPP`] based on frequency. + pub fn opp_from_freq( + &self, + freq: Hertz, + available: Option, + index: Option, + stype: SearchType, + ) -> Result> { + let raw_dev = self.dev.as_raw(); + let index = index.unwrap_or(0); + let mut rate = freq.into(); + + let ptr = from_err_ptr(match stype { + SearchType::Exact => { + if let Some(available) = available { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and + // its safety requirements. The returned pointer will be owned by the new + // [`OPP`] instance. + unsafe { + bindings::dev_pm_opp_find_freq_exact_indexed( + raw_dev, rate, index, available, + ) + } + } else { + return Err(EINVAL); + } + } + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_freq_ceil_indexed(raw_dev, &mut rate, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_freq_floor_indexed(raw_dev, &mut rate, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on level. + pub fn opp_from_level(&self, mut level: u32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Exact => unsafe { bindings::dev_pm_opp_find_level_exact(raw_dev, level) }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_level_ceil(raw_dev, &mut level) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_level_floor(raw_dev, &mut level) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on bandwidth. + pub fn opp_from_bw(&self, mut bw: u32, index: i32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // The OPP core doesn't support this yet. + SearchType::Exact => return Err(EINVAL), + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_bw_ceil(raw_dev, &mut bw, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_bw_floor(raw_dev, &mut bw, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Enables the [`OPP`]. + #[inline] + pub fn enable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_enable(self.dev.as_raw(), freq.into()) }) + } + + /// Disables the [`OPP`]. + #[inline] + pub fn disable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_disable(self.dev.as_raw(), freq.into()) }) + } + + /// Registers with the Energy model. + #[cfg(CONFIG_OF)] + pub fn of_register_em(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_register_em(self.dev.as_raw(), cpumask.as_raw()) + })?; + + self.em = true; + Ok(()) + } + + /// Unregisters with the Energy model. + #[cfg(all(CONFIG_OF, CONFIG_ENERGY_MODEL))] + #[inline] + fn of_unregister_em(&self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. We registered with the EM framework earlier, it is safe to unregister now. + unsafe { bindings::em_dev_unregister_perf_domain(self.dev.as_raw()) }; + } +} + +impl Drop for Table { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe + // to relinquish it now. + unsafe { bindings::dev_pm_opp_put_opp_table(self.ptr) }; + + #[cfg(CONFIG_OF)] + { + #[cfg(CONFIG_ENERGY_MODEL)] + if self.em { + self.of_unregister_em(); + } + + if self.of { + self.remove_of(); + } else if let Some(cpumask) = self.cpus.take() { + self.remove_of_cpumask(&cpumask); + } + } + } +} + +/// A reference-counted Operating performance point (OPP). +/// +/// Rust abstraction for the C `struct dev_pm_opp`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`OPP`]. +/// +/// Instances of this type are reference-counted. The reference count is incremented by the +/// `dev_pm_opp_get` function and decremented by `dev_pm_opp_put`. The Rust type `ARef` +/// represents a pointer that owns a reference count on the [`OPP`]. +/// +/// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code. +/// +/// # Examples +/// +/// The following example demonstrates how to get [`OPP`] corresponding to a frequency value and +/// configure the device with it. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{SearchType, Table}; +/// +/// fn configure_opp(table: &Table, freq: Hertz) -> Result { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// if opp.freq(None) != freq { +/// return Err(EINVAL); +/// } +/// +/// table.set_opp(&opp) +/// } +/// ``` +#[repr(transparent)] +pub struct OPP(Opaque); + +/// SAFETY: It is okay to send the ownership of [`OPP`] across thread boundaries. +unsafe impl Send for OPP {} + +/// SAFETY: It is okay to access [`OPP`] through shared references from other threads because we're +/// either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for OPP {} + +/// SAFETY: The type invariants guarantee that [`OPP`] is always refcounted. +unsafe impl AlwaysRefCounted for OPP { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_get(self.0.get()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_put(obj.cast().as_ptr()) } + } +} + +impl OPP { + /// Creates an owned reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is incremented by the C code and will be decremented by `dec_ref` when the + /// [`ARef`] object is dropped. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and the refcount of the [`OPP`] is incremented. + /// The caller must also ensure that it doesn't explicitly drop the refcount of the [`OPP`], as + /// the returned [`ARef`] object takes over the refcount increment on the underlying object and + /// the same will be dropped along with it. + pub unsafe fn from_raw_opp_owned(ptr: *mut bindings::dev_pm_opp) -> Result> { + let ptr = ptr::NonNull::new(ptr).ok_or(ENODEV)?; + + // SAFETY: The safety requirements guarantee the validity of the pointer. + // + // INVARIANT: The reference-count is decremented when [`OPP`] goes out of scope. + Ok(unsafe { ARef::from_raw(ptr.cast()) }) + } + + /// Creates a reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is not updated by the Rust API unless the returned reference is converted to + /// an [`ARef`] object. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the duration of `'a`. + #[inline] + pub unsafe fn from_raw_opp<'a>(ptr: *mut bindings::dev_pm_opp) -> Result<&'a Self> { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of 'a. The cast is okay because [`OPP`] is `repr(transparent)`. + Ok(unsafe { &*ptr.cast() }) + } + + #[inline] + fn as_raw(&self) -> *mut bindings::dev_pm_opp { + self.0.get() + } + + /// Returns the frequency of an [`OPP`]. + #[inline] + pub fn freq(&self, index: Option) -> Hertz { + let index = index.unwrap_or(0); + + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + Hertz(unsafe { bindings::dev_pm_opp_get_freq_indexed(self.as_raw(), index) }) + } + + /// Returns the voltage of an [`OPP`]. + #[inline] + pub fn voltage(&self) -> MicroVolt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroVolt(unsafe { bindings::dev_pm_opp_get_voltage(self.as_raw()) }) + } + + /// Returns the level of an [`OPP`]. + #[inline] + pub fn level(&self) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_level(self.as_raw()) } + } + + /// Returns the power of an [`OPP`]. + #[inline] + pub fn power(&self) -> MicroWatt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroWatt(unsafe { bindings::dev_pm_opp_get_power(self.as_raw()) }) + } + + /// Returns the required pstate of an [`OPP`]. + #[inline] + pub fn required_pstate(&self, index: u32) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_required_pstate(self.as_raw(), index) } + } + + /// Returns true if the [`OPP`] is turbo. + #[inline] + pub fn is_turbo(&self) -> bool { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_is_turbo(self.as_raw()) } + } +} diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs new file mode 100644 index 000000000..1c0796ea2 --- /dev/null +++ b/rust/kernel/page.rs @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel page allocation and management. + +use crate::{ + alloc::{ + AllocError, + Flags, // + }, + bindings, + error::{ + code::*, + Result, // + }, + uaccess::UserSliceReader, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::{ + self, + NonNull, // + }, // +}; + +/// A bitwise shift for the page size. +pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; + +/// The number of bytes in a page. +pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; + +/// A bitmask that gives the page containing a given address. +pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); + +/// Rounds up to the next multiple of [`PAGE_SIZE`]. +/// +/// Returns [`None`] on integer overflow. +/// +/// # Examples +/// +/// ``` +/// use kernel::page::{ +/// page_align, +/// PAGE_SIZE, +/// }; +/// +/// // Requested address is already aligned. +/// assert_eq!(page_align(0x0), Some(0x0)); +/// assert_eq!(page_align(PAGE_SIZE), Some(PAGE_SIZE)); +/// +/// // Requested address needs alignment up. +/// assert_eq!(page_align(0x1), Some(PAGE_SIZE)); +/// assert_eq!(page_align(PAGE_SIZE + 1), Some(2 * PAGE_SIZE)); +/// +/// // Requested address causes overflow (returns `None`). +/// let overflow_addr = usize::MAX - (PAGE_SIZE / 2); +/// assert_eq!(page_align(overflow_addr), None); +/// ``` +#[inline(always)] +pub const fn page_align(addr: usize) -> Option { + let Some(sum) = addr.checked_add(PAGE_SIZE - 1) else { + return None; + }; + Some(sum & PAGE_MASK) +} + +/// Representation of a non-owning reference to a [`Page`]. +/// +/// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a +/// [`Vmalloc`] allocation such as [`VBox`]. +/// +/// # Example +/// +/// ``` +/// # use kernel::{bindings, prelude::*}; +/// use kernel::page::{BorrowedPage, Page, PAGE_SIZE}; +/// # use core::{mem::MaybeUninit, ptr, ptr::NonNull }; +/// +/// fn borrow_page<'a>(vbox: &'a mut VBox>) -> BorrowedPage<'a> { +/// let ptr = ptr::from_ref(&**vbox); +/// +/// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. +/// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) }; +/// +/// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid +/// // pointer to `Vmalloc` memory. +/// let page = unsafe { NonNull::new_unchecked(page) }; +/// +/// // SAFETY: +/// // - `self.0` is a valid pointer to a `struct page`. +/// // - `self.0` is valid for the entire lifetime of `self`. +/// unsafe { BorrowedPage::from_raw(page) } +/// } +/// +/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; +/// let page = borrow_page(&mut vbox); +/// +/// // SAFETY: There is no concurrent read or write to this page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`. +/// +/// [`VBox`]: kernel::alloc::VBox +/// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc +pub struct BorrowedPage<'a>(ManuallyDrop, PhantomData<&'a Page>); + +impl<'a> BorrowedPage<'a> { + /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. + /// + /// # Safety + /// + /// - `ptr` must point to a valid `bindings::page`. + /// - `ptr` must remain valid for the entire lifetime `'a`. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + let page = Page { page: ptr }; + + // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime + // `'a`. + Self(ManuallyDrop::new(page), PhantomData) + } +} + +impl<'a> Deref for BorrowedPage<'a> { + type Target = Page; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Trait to be implemented by types which provide an [`Iterator`] implementation of +/// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). +pub trait AsPageIter { + /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. + fn page_iter(&mut self) -> Self::Iter<'_>; +} + +/// A pointer to a page that owns the page allocation. +/// +/// # Invariants +/// +/// The pointer is valid, and has ownership over the page. +pub struct Page { + page: NonNull, +} + +// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across +// threads is safe. +unsafe impl Send for Page {} + +// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing +// them concurrently is safe. +unsafe impl Sync for Page {} + +impl Page { + /// Allocates a new page. + /// + /// # Examples + /// + /// Allocate memory for a page. + /// + /// ``` + /// use kernel::page::Page; + /// + /// let page = Page::alloc_page(GFP_KERNEL)?; + /// # Ok::<(), kernel::alloc::AllocError>(()) + /// ``` + /// + /// Allocate memory for a page and zero its contents. + /// + /// ``` + /// use kernel::page::Page; + /// + /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + /// # Ok::<(), kernel::alloc::AllocError>(()) + /// ``` + #[inline] + pub fn alloc_page(flags: Flags) -> Result { + // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it + // is always safe to call this method. + let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; + let page = NonNull::new(page).ok_or(AllocError)?; + // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly + // allocated page. We transfer that ownership to the new `Page` object. + Ok(Self { page }) + } + + /// Returns a raw pointer to the page. + pub fn as_ptr(&self) -> *mut bindings::page { + self.page.as_ptr() + } + + /// Get the node id containing this page. + #[inline] + pub fn nid(&self) -> i32 { + // SAFETY: Always safe to call with a valid page. + unsafe { bindings::page_to_nid(self.as_ptr()) } + } + + /// Runs a piece of code with this page mapped to an address. + /// + /// The page is unmapped when this call returns. + /// + /// # Using the raw pointer + /// + /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for + /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might + /// only be mapped on the current thread, and when that is the case, dereferencing it on other + /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't + /// cause data races, the memory may be uninitialized, and so on. + /// + /// If multiple threads map the same page at the same time, then they may reference with + /// different addresses. However, even if the addresses are different, the underlying memory is + /// still the same for these purposes (e.g., it's still a data race if they both write to the + /// same underlying byte at the same time). + fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { + // SAFETY: `page` is valid due to the type invariants on `Page`. + let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; + + let res = f(mapped_addr.cast()); + + // This unmaps the page mapped above. + // + // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner + // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. + // + // In other words, if this call to `kunmap_local` happens when a different page should be + // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the + // call just above in `with_page_mapped` that made that possible. In this case, it is the + // unsafe block that wraps that other call that is incorrect. + unsafe { bindings::kunmap_local(mapped_addr) }; + + res + } + + /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. + /// + /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the + /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on + /// this task, as this method uses a local mapping. + /// + /// If `off` and `len` refers to a region outside of this page, then this method returns + /// [`EINVAL`] and does not call `f`. + /// + /// # Using the raw pointer + /// + /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for + /// `len` bytes and for the duration in which the closure is called. The pointer might only be + /// mapped on the current thread, and when that is the case, dereferencing it on other threads + /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause + /// data races, the memory may be uninitialized, and so on. + /// + /// If multiple threads map the same page at the same time, then they may reference with + /// different addresses. However, even if the addresses are different, the underlying memory is + /// still the same for these purposes (e.g., it's still a data race if they both write to the + /// same underlying byte at the same time). + fn with_pointer_into_page( + &self, + off: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, + ) -> Result { + let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; + + if bounds_ok { + self.with_page_mapped(move |page_addr| { + // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will + // result in a pointer that is in bounds or one off the end of the page. + f(unsafe { page_addr.add(off) }) + }) + } else { + Err(EINVAL) + } + } + + /// Maps the page and reads from it into the given buffer. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// * Callers must ensure that `dst` is valid for writing `len` bytes. + /// * Callers must ensure that this call does not race with a write to the same page that + /// overlaps with this read. + pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |src| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then + // it has performed a bounds check and guarantees that `src` is + // valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::copy_nonoverlapping(src, dst, len) }; + Ok(()) + }) + } + + /// Maps the page and writes into it from the given buffer. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// * Callers must ensure that `src` is valid for reading `len` bytes. + /// * Callers must ensure that this call does not race with a read or write to the same page + /// that overlaps with this write. + pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::copy_nonoverlapping(src, dst, len) }; + Ok(()) + }) + } + + /// Maps the page and zeroes the given slice. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// Callers must ensure that this call does not race with a read or write to the same page that + /// overlaps with this write. + pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::write_bytes(dst, 0u8, len) }; + Ok(()) + }) + } + + /// Copies data from userspace into this page. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. + /// However, they are not allowed on the page you are copying into. + /// + /// # Safety + /// + /// Callers must ensure that this call does not race with a read or write to the same page that + /// overlaps with this write. + pub unsafe fn copy_from_user_slice_raw( + &self, + reader: &mut UserSliceReader, + offset: usize, + len: usize, + ) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have + // exclusive access to the slice since the caller guarantees that there are no races. + reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) + }) + } +} + +impl Drop for Page { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, we have ownership of the page and can free it. + unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; + } +} diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs new file mode 100644 index 000000000..3ec897709 --- /dev/null +++ b/rust/kernel/pci.rs @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the PCI bus. +//! +//! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) + +use crate::{ + bindings, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + str::CStr, + types::Opaque, + ThisModule, // +}; +use core::{ + marker::PhantomData, + mem::offset_of, + num::NonZero, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +mod id; +mod io; +mod irq; + +pub use self::id::{ + Class, + ClassMask, + Vendor, // +}; +pub use self::io::{ + Bar, + ConfigSpace, + ConfigSpaceSize, + DevresBar, + Extended, + Normal, // +}; +pub use self::irq::{ + IrqType, + IrqTypes, + IrqVector, + IrqVectorRegistration, // +}; + +/// An adapter for the registration of PCI drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::pci_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct pci_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::pci_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + pdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct pci_driver` on initialization. + unsafe { + (*pdrv.get()).name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__pci_register_driver(pdrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(pdrv: &Opaque) { + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::pci_unregister_driver(pdrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback( + pdev: *mut bindings::pci_dev, + id: *const bindings::pci_device_id, + ) -> c_int { + // SAFETY: The PCI bus only ever calls the probe callback with a valid pointer to a + // `struct pci_dev`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>` or + // `pci_device_id_any` which has 0 as driver_data. It can also come from dynamic IDs, which + // will ensure that `driver_data` exists in `T::ID_TABLE`. + let info = unsafe { id.info_unchecked_opt::() }; + + from_result(|| { + let data = T::probe(pdev, info); + + pdev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { + // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a + // `struct pci_dev`. + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; + + T::unbind(pdev, data); + } +} + +/// Declares a kernel module that exposes a single PCI driver. +/// +/// # Examples +/// +///```ignore +/// kernel::module_pci_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +///``` +#[macro_export] +macro_rules! module_pci_driver { +($($f:tt)*) => { + $crate::module_driver!(, $crate::pci::Adapter, { $($f)* }); +}; +} + +/// Abstraction for the PCI device ID structure ([`struct pci_device_id`]). +/// +/// [`struct pci_device_id`]: https://docs.kernel.org/PCI/pci.html#c.pci_device_id +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::pci_device_id); + +impl DeviceId { + const PCI_ANY_ID: u32 = !0; + + /// Equivalent to C's `PCI_DEVICE` macro. + /// + /// Create a new `pci::DeviceId` from a vendor and device ID. + #[inline] + pub const fn from_id(vendor: Vendor, device: u32) -> Self { + Self(bindings::pci_device_id { + vendor: vendor.as_raw() as u32, + device, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class: 0, + class_mask: 0, + driver_data: 0, + override_only: 0, + }) + } + + /// Equivalent to C's `PCI_DEVICE_CLASS` macro. + /// + /// Create a new `pci::DeviceId` from a class number and mask. + #[inline] + pub const fn from_class(class: u32, class_mask: u32) -> Self { + Self(bindings::pci_device_id { + vendor: DeviceId::PCI_ANY_ID, + device: DeviceId::PCI_ANY_ID, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class, + class_mask, + driver_data: 0, + override_only: 0, + }) + } + + /// Create a new [`DeviceId`] from a class number, mask, and specific vendor. + /// + /// This is more targeted than [`DeviceId::from_class`]: in addition to matching by [`Vendor`], + /// it also matches the PCI [`Class`] (up to the entire 24 bits, depending on the + /// [`ClassMask`]). + #[inline] + pub const fn from_class_and_vendor( + class: Class, + class_mask: ClassMask, + vendor: Vendor, + ) -> Self { + Self(bindings::pci_device_id { + vendor: vendor.as_raw() as u32, + device: DeviceId::PCI_ANY_ID, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class: class.as_raw(), + class_mask: class_mask.as_raw(), + driver_data: 0, + override_only: 0, + }) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `pci_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::pci_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); +} + +/// `IdTable` type for PCI. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a PCI `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! pci_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("pci", $crate::pci::DeviceId, $($tt)*); + }; +} + +/// The PCI driver trait. +/// +/// # Examples +/// +///``` +/// # use kernel::{bindings, device::Core, pci}; +/// +/// struct MyDriver; +/// +/// kernel::pci_device_table!( +/// PCI_TABLE, +/// ::IdInfo, +/// [ +/// ( +/// pci::DeviceId::from_id(pci::Vendor::REDHAT, bindings::PCI_ANY_ID as u32), +/// (), +/// ) +/// ] +/// ); +/// +/// impl pci::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const ID_TABLE: pci::IdTable = &PCI_TABLE; +/// +/// fn probe<'bound>( +/// _pdev: &'bound pci::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// } +///``` +/// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the +/// `Adapter` documentation for an example. +pub trait Driver { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// PCI driver probe. + /// + /// Called when a new pci device is added or discovered. Implementers should + /// attempt to initialize the device here. + fn probe<'bound>( + dev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// PCI driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The PCI device representation. +/// +/// This structure represents the Rust abstraction for a C `struct pci_dev`. The implementation +/// abstracts the usage of an already existing C `struct pci_dev` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct pci_dev` created by the C portion of the +/// kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::pci_dev { + self.0.get() + } +} + +impl Device { + /// Returns the PCI vendor ID as [`Vendor`]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{device::Core, pci::{self, Vendor}, prelude::*}; + /// fn log_device_info(pdev: &pci::Device>) -> Result { + /// // Get an instance of `Vendor`. + /// let vendor = pdev.vendor_id(); + /// dev_info!( + /// pdev, + /// "Device: Vendor={}, Device=0x{:x}\n", + /// vendor, + /// pdev.device_id() + /// ); + /// Ok(()) + /// } + /// ``` + #[inline] + pub fn vendor_id(&self) -> Vendor { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + let vendor_id = unsafe { (*self.as_raw()).vendor }; + Vendor::from_raw(vendor_id) + } + + /// Returns the PCI device ID. + #[inline] + pub fn device_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).device } + } + + /// Returns the PCI revision ID. + #[inline] + pub fn revision_id(&self) -> u8 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).revision } + } + + /// Returns the PCI bus device/function. + #[inline] + pub fn dev_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { bindings::pci_dev_id(self.as_raw()) } + } + + /// Returns the PCI subsystem vendor ID. + #[inline] + pub fn subsystem_vendor_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).subsystem_vendor } + } + + /// Returns the PCI subsystem device ID. + #[inline] + pub fn subsystem_device_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).subsystem_device } + } + + /// Returns the start of the given PCI BAR resource. + pub fn resource_start(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + Ok(unsafe { bindings::pci_resource_start(self.as_raw(), bar.try_into()?) }) + } + + /// Returns the size of the given PCI BAR resource. + pub fn resource_len(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) + } + + /// Returns the PCI class as a `Class` struct. + #[inline] + pub fn pci_class(&self) -> Class { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + Class::from_raw(unsafe { (*self.as_raw()).class }) + } +} + +impl<'a> Device> { + /// Returns the total number of VFs, or [`None`] if SR-IOV is not available. + #[inline] + pub fn sriov_get_totalvfs(&self) -> Option> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct pci_dev`. + let total_vfs = unsafe { bindings::pci_sriov_get_totalvfs(self.as_raw()) }; + + // CAST: The C function returns `unsigned int`, but the value originates + // from TotalVFs/driver_max_VFs (which are defined as `u16`), so this cast + // cannot truncate. + NonZero::new(total_vfs as u16) + } + + /// Enable memory resources for this device. + pub fn enable_device_mem(&self) -> Result { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + to_result(unsafe { bindings::pci_enable_device_mem(self.as_raw()) }) + } + + /// Enable bus-mastering for this device. + #[inline] + pub fn set_master(&self) { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + unsafe { bindings::pci_set_master(self.as_raw()) }; + } +} + +// SAFETY: `pci::Device` is a transparent wrapper of `struct pci_dev`. +// The offset is guaranteed to point to a valid device field inside `pci::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::pci_dev, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +impl<'a> crate::dma::Device<'a> for Device> {} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::pci_dev_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct pci_dev`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &Device { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if !unsafe { bindings::dev_is_pci(dev.as_raw()) } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the bus type of `dev` equals `bindings::pci_bus_type`, + // hence `dev` must be embedded in a valid `struct pci_dev` as guaranteed by the + // corresponding C code. + let pdev = unsafe { container_of!(dev.as_raw(), bindings::pci_dev, dev) }; + + // SAFETY: `pdev` is a valid pointer to a `struct pci_dev`. + Ok(unsafe { &*pdev.cast() }) + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct pci_dev` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs new file mode 100644 index 000000000..dbaf30166 --- /dev/null +++ b/rust/kernel/pci/id.rs @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI device identifiers and related types. +//! +//! This module contains PCI class codes, Vendor IDs, and supporting types. + +use crate::{ + bindings, + fmt, + prelude::*, // +}; + +/// PCI device class codes. +/// +/// Each entry contains the full 24-bit PCI class code (base class in bits +/// 23-16, subclass in bits 15-8, programming interface in bits 7-0). +/// +/// # Examples +/// +/// ``` +/// # use kernel::{device::Core, pci::{self, Class}, prelude::*}; +/// fn probe_device(pdev: &pci::Device>) -> Result { +/// let pci_class = pdev.pci_class(); +/// dev_info!( +/// pdev, +/// "Detected PCI class: {}\n", +/// pci_class +/// ); +/// Ok(()) +/// } +/// ``` +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct Class(u32); + +/// PCI class mask constants for matching [`Class`] codes. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClassMask { + /// Match the full 24-bit class code. + Full = 0xffffff, + /// Match the upper 16 bits of the class code (base class and subclass only) + ClassSubclass = 0xffff00, +} + +macro_rules! define_all_pci_classes { + ( + $($variant:ident = $binding:expr,)+ + ) => { + impl Class { + $( + #[allow(missing_docs)] + pub const $variant: Self = Self(Self::to_24bit_class($binding)); + )+ + } + + impl fmt::Display for Class { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + $( + &Self::$variant => write!(f, stringify!($variant)), + )+ + _ => ::fmt(self, f), + } + } + } + }; +} + +/// Once constructed, a [`Class`] contains a valid PCI class code. +impl Class { + /// Create a [`Class`] from a raw 24-bit class code. + #[inline] + pub(super) fn from_raw(class_code: u32) -> Self { + Self(class_code) + } + + /// Get the raw 24-bit class code value. + #[inline] + pub const fn as_raw(self) -> u32 { + self.0 + } + + // Converts a PCI class constant to 24-bit format. + // + // Many device drivers use only the upper 16 bits (base class and subclass), + // but some use the full 24 bits. In order to support both cases, store the + // class code as a 24-bit value, where 16-bit values are shifted up 8 bits. + const fn to_24bit_class(val: u32) -> u32 { + if val > 0xFFFF { + val + } else { + val << 8 + } + } +} + +impl fmt::Debug for Class { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "0x{:06x}", self.0) + } +} + +impl ClassMask { + /// Get the raw mask value. + #[inline] + pub const fn as_raw(self) -> u32 { + self as u32 + } +} + +impl TryFrom for ClassMask { + type Error = Error; + + fn try_from(value: u32) -> Result { + match value { + 0xffffff => Ok(ClassMask::Full), + 0xffff00 => Ok(ClassMask::ClassSubclass), + _ => Err(EINVAL), + } + } +} + +/// PCI vendor IDs. +/// +/// Each entry contains the 16-bit PCI vendor ID as assigned by the PCI SIG. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct Vendor(u16); + +macro_rules! define_all_pci_vendors { + ( + $($variant:ident = $binding:expr,)+ + ) => { + impl Vendor { + $( + #[allow(missing_docs)] + pub const $variant: Self = Self($binding as u16); + )+ + } + + impl fmt::Display for Vendor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + $( + &Self::$variant => write!(f, stringify!($variant)), + )+ + _ => ::fmt(self, f), + } + } + } + }; +} + +/// Once constructed, a `Vendor` contains a valid PCI Vendor ID. +impl Vendor { + /// Create a Vendor from a raw 16-bit vendor ID. + #[inline] + pub(super) fn from_raw(vendor_id: u16) -> Self { + Self(vendor_id) + } + + /// Get the raw 16-bit vendor ID value. + #[inline] + pub const fn as_raw(self) -> u16 { + self.0 + } +} + +impl fmt::Debug for Vendor { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "0x{:04x}", self.0) + } +} + +define_all_pci_classes! { + NOT_DEFINED = bindings::PCI_CLASS_NOT_DEFINED, // 0x000000 + NOT_DEFINED_VGA = bindings::PCI_CLASS_NOT_DEFINED_VGA, // 0x000100 + + STORAGE_SCSI = bindings::PCI_CLASS_STORAGE_SCSI, // 0x010000 + STORAGE_IDE = bindings::PCI_CLASS_STORAGE_IDE, // 0x010100 + STORAGE_FLOPPY = bindings::PCI_CLASS_STORAGE_FLOPPY, // 0x010200 + STORAGE_IPI = bindings::PCI_CLASS_STORAGE_IPI, // 0x010300 + STORAGE_RAID = bindings::PCI_CLASS_STORAGE_RAID, // 0x010400 + STORAGE_SATA = bindings::PCI_CLASS_STORAGE_SATA, // 0x010600 + STORAGE_SATA_AHCI = bindings::PCI_CLASS_STORAGE_SATA_AHCI, // 0x010601 + STORAGE_SAS = bindings::PCI_CLASS_STORAGE_SAS, // 0x010700 + STORAGE_EXPRESS = bindings::PCI_CLASS_STORAGE_EXPRESS, // 0x010802 + STORAGE_OTHER = bindings::PCI_CLASS_STORAGE_OTHER, // 0x018000 + + NETWORK_ETHERNET = bindings::PCI_CLASS_NETWORK_ETHERNET, // 0x020000 + NETWORK_TOKEN_RING = bindings::PCI_CLASS_NETWORK_TOKEN_RING, // 0x020100 + NETWORK_FDDI = bindings::PCI_CLASS_NETWORK_FDDI, // 0x020200 + NETWORK_ATM = bindings::PCI_CLASS_NETWORK_ATM, // 0x020300 + NETWORK_OTHER = bindings::PCI_CLASS_NETWORK_OTHER, // 0x028000 + + DISPLAY_VGA = bindings::PCI_CLASS_DISPLAY_VGA, // 0x030000 + DISPLAY_XGA = bindings::PCI_CLASS_DISPLAY_XGA, // 0x030100 + DISPLAY_3D = bindings::PCI_CLASS_DISPLAY_3D, // 0x030200 + DISPLAY_OTHER = bindings::PCI_CLASS_DISPLAY_OTHER, // 0x038000 + + MULTIMEDIA_VIDEO = bindings::PCI_CLASS_MULTIMEDIA_VIDEO, // 0x040000 + MULTIMEDIA_AUDIO = bindings::PCI_CLASS_MULTIMEDIA_AUDIO, // 0x040100 + MULTIMEDIA_PHONE = bindings::PCI_CLASS_MULTIMEDIA_PHONE, // 0x040200 + MULTIMEDIA_HD_AUDIO = bindings::PCI_CLASS_MULTIMEDIA_HD_AUDIO, // 0x040300 + MULTIMEDIA_OTHER = bindings::PCI_CLASS_MULTIMEDIA_OTHER, // 0x048000 + + MEMORY_RAM = bindings::PCI_CLASS_MEMORY_RAM, // 0x050000 + MEMORY_FLASH = bindings::PCI_CLASS_MEMORY_FLASH, // 0x050100 + MEMORY_CXL = bindings::PCI_CLASS_MEMORY_CXL, // 0x050200 + MEMORY_OTHER = bindings::PCI_CLASS_MEMORY_OTHER, // 0x058000 + + BRIDGE_HOST = bindings::PCI_CLASS_BRIDGE_HOST, // 0x060000 + BRIDGE_ISA = bindings::PCI_CLASS_BRIDGE_ISA, // 0x060100 + BRIDGE_EISA = bindings::PCI_CLASS_BRIDGE_EISA, // 0x060200 + BRIDGE_MC = bindings::PCI_CLASS_BRIDGE_MC, // 0x060300 + BRIDGE_PCI_NORMAL = bindings::PCI_CLASS_BRIDGE_PCI_NORMAL, // 0x060400 + BRIDGE_PCI_SUBTRACTIVE = bindings::PCI_CLASS_BRIDGE_PCI_SUBTRACTIVE, // 0x060401 + BRIDGE_PCMCIA = bindings::PCI_CLASS_BRIDGE_PCMCIA, // 0x060500 + BRIDGE_NUBUS = bindings::PCI_CLASS_BRIDGE_NUBUS, // 0x060600 + BRIDGE_CARDBUS = bindings::PCI_CLASS_BRIDGE_CARDBUS, // 0x060700 + BRIDGE_RACEWAY = bindings::PCI_CLASS_BRIDGE_RACEWAY, // 0x060800 + BRIDGE_OTHER = bindings::PCI_CLASS_BRIDGE_OTHER, // 0x068000 + + COMMUNICATION_SERIAL = bindings::PCI_CLASS_COMMUNICATION_SERIAL, // 0x070000 + COMMUNICATION_PARALLEL = bindings::PCI_CLASS_COMMUNICATION_PARALLEL, // 0x070100 + COMMUNICATION_MULTISERIAL = bindings::PCI_CLASS_COMMUNICATION_MULTISERIAL, // 0x070200 + COMMUNICATION_MODEM = bindings::PCI_CLASS_COMMUNICATION_MODEM, // 0x070300 + COMMUNICATION_OTHER = bindings::PCI_CLASS_COMMUNICATION_OTHER, // 0x078000 + + SYSTEM_PIC = bindings::PCI_CLASS_SYSTEM_PIC, // 0x080000 + SYSTEM_PIC_IOAPIC = bindings::PCI_CLASS_SYSTEM_PIC_IOAPIC, // 0x080010 + SYSTEM_PIC_IOXAPIC = bindings::PCI_CLASS_SYSTEM_PIC_IOXAPIC, // 0x080020 + SYSTEM_DMA = bindings::PCI_CLASS_SYSTEM_DMA, // 0x080100 + SYSTEM_TIMER = bindings::PCI_CLASS_SYSTEM_TIMER, // 0x080200 + SYSTEM_RTC = bindings::PCI_CLASS_SYSTEM_RTC, // 0x080300 + SYSTEM_PCI_HOTPLUG = bindings::PCI_CLASS_SYSTEM_PCI_HOTPLUG, // 0x080400 + SYSTEM_SDHCI = bindings::PCI_CLASS_SYSTEM_SDHCI, // 0x080500 + SYSTEM_RCEC = bindings::PCI_CLASS_SYSTEM_RCEC, // 0x080700 + SYSTEM_OTHER = bindings::PCI_CLASS_SYSTEM_OTHER, // 0x088000 + + INPUT_KEYBOARD = bindings::PCI_CLASS_INPUT_KEYBOARD, // 0x090000 + INPUT_PEN = bindings::PCI_CLASS_INPUT_PEN, // 0x090100 + INPUT_MOUSE = bindings::PCI_CLASS_INPUT_MOUSE, // 0x090200 + INPUT_SCANNER = bindings::PCI_CLASS_INPUT_SCANNER, // 0x090300 + INPUT_GAMEPORT = bindings::PCI_CLASS_INPUT_GAMEPORT, // 0x090400 + INPUT_OTHER = bindings::PCI_CLASS_INPUT_OTHER, // 0x098000 + + DOCKING_GENERIC = bindings::PCI_CLASS_DOCKING_GENERIC, // 0x0a0000 + DOCKING_OTHER = bindings::PCI_CLASS_DOCKING_OTHER, // 0x0a8000 + + PROCESSOR_386 = bindings::PCI_CLASS_PROCESSOR_386, // 0x0b0000 + PROCESSOR_486 = bindings::PCI_CLASS_PROCESSOR_486, // 0x0b0100 + PROCESSOR_PENTIUM = bindings::PCI_CLASS_PROCESSOR_PENTIUM, // 0x0b0200 + PROCESSOR_ALPHA = bindings::PCI_CLASS_PROCESSOR_ALPHA, // 0x0b1000 + PROCESSOR_POWERPC = bindings::PCI_CLASS_PROCESSOR_POWERPC, // 0x0b2000 + PROCESSOR_MIPS = bindings::PCI_CLASS_PROCESSOR_MIPS, // 0x0b3000 + PROCESSOR_CO = bindings::PCI_CLASS_PROCESSOR_CO, // 0x0b4000 + + SERIAL_FIREWIRE = bindings::PCI_CLASS_SERIAL_FIREWIRE, // 0x0c0000 + SERIAL_FIREWIRE_OHCI = bindings::PCI_CLASS_SERIAL_FIREWIRE_OHCI, // 0x0c0010 + SERIAL_ACCESS = bindings::PCI_CLASS_SERIAL_ACCESS, // 0x0c0100 + SERIAL_SSA = bindings::PCI_CLASS_SERIAL_SSA, // 0x0c0200 + SERIAL_USB_UHCI = bindings::PCI_CLASS_SERIAL_USB_UHCI, // 0x0c0300 + SERIAL_USB_OHCI = bindings::PCI_CLASS_SERIAL_USB_OHCI, // 0x0c0310 + SERIAL_USB_EHCI = bindings::PCI_CLASS_SERIAL_USB_EHCI, // 0x0c0320 + SERIAL_USB_XHCI = bindings::PCI_CLASS_SERIAL_USB_XHCI, // 0x0c0330 + SERIAL_USB_CDNS = bindings::PCI_CLASS_SERIAL_USB_CDNS, // 0x0c0380 + SERIAL_USB_DEVICE = bindings::PCI_CLASS_SERIAL_USB_DEVICE, // 0x0c03fe + SERIAL_FIBER = bindings::PCI_CLASS_SERIAL_FIBER, // 0x0c0400 + SERIAL_SMBUS = bindings::PCI_CLASS_SERIAL_SMBUS, // 0x0c0500 + SERIAL_IPMI_SMIC = bindings::PCI_CLASS_SERIAL_IPMI_SMIC, // 0x0c0700 + SERIAL_IPMI_KCS = bindings::PCI_CLASS_SERIAL_IPMI_KCS, // 0x0c0701 + SERIAL_IPMI_BT = bindings::PCI_CLASS_SERIAL_IPMI_BT, // 0x0c0702 + + WIRELESS_RF_CONTROLLER = bindings::PCI_CLASS_WIRELESS_RF_CONTROLLER, // 0x0d1000 + WIRELESS_WHCI = bindings::PCI_CLASS_WIRELESS_WHCI, // 0x0d1010 + + INTELLIGENT_I2O = bindings::PCI_CLASS_INTELLIGENT_I2O, // 0x0e0000 + + SATELLITE_TV = bindings::PCI_CLASS_SATELLITE_TV, // 0x0f0000 + SATELLITE_AUDIO = bindings::PCI_CLASS_SATELLITE_AUDIO, // 0x0f0100 + SATELLITE_VOICE = bindings::PCI_CLASS_SATELLITE_VOICE, // 0x0f0300 + SATELLITE_DATA = bindings::PCI_CLASS_SATELLITE_DATA, // 0x0f0400 + + CRYPT_NETWORK = bindings::PCI_CLASS_CRYPT_NETWORK, // 0x100000 + CRYPT_ENTERTAINMENT = bindings::PCI_CLASS_CRYPT_ENTERTAINMENT, // 0x100100 + CRYPT_OTHER = bindings::PCI_CLASS_CRYPT_OTHER, // 0x108000 + + SP_DPIO = bindings::PCI_CLASS_SP_DPIO, // 0x110000 + SP_OTHER = bindings::PCI_CLASS_SP_OTHER, // 0x118000 + + ACCELERATOR_PROCESSING = bindings::PCI_CLASS_ACCELERATOR_PROCESSING, // 0x120000 + + OTHERS = bindings::PCI_CLASS_OTHERS, // 0xff0000 +} + +define_all_pci_vendors! { + PCI_SIG = bindings::PCI_VENDOR_ID_PCI_SIG, // 0x0001 + LOONGSON = bindings::PCI_VENDOR_ID_LOONGSON, // 0x0014 + SOLIDIGM = bindings::PCI_VENDOR_ID_SOLIDIGM, // 0x025e + TTTECH = bindings::PCI_VENDOR_ID_TTTECH, // 0x0357 + DYNALINK = bindings::PCI_VENDOR_ID_DYNALINK, // 0x0675 + UBIQUITI = bindings::PCI_VENDOR_ID_UBIQUITI, // 0x0777 + BERKOM = bindings::PCI_VENDOR_ID_BERKOM, // 0x0871 + ITTIM = bindings::PCI_VENDOR_ID_ITTIM, // 0x0b48 + COMPAQ = bindings::PCI_VENDOR_ID_COMPAQ, // 0x0e11 + LSI_LOGIC = bindings::PCI_VENDOR_ID_LSI_LOGIC, // 0x1000 + ATI = bindings::PCI_VENDOR_ID_ATI, // 0x1002 + VLSI = bindings::PCI_VENDOR_ID_VLSI, // 0x1004 + ADL = bindings::PCI_VENDOR_ID_ADL, // 0x1005 + NS = bindings::PCI_VENDOR_ID_NS, // 0x100b + TSENG = bindings::PCI_VENDOR_ID_TSENG, // 0x100c + WEITEK = bindings::PCI_VENDOR_ID_WEITEK, // 0x100e + DEC = bindings::PCI_VENDOR_ID_DEC, // 0x1011 + CIRRUS = bindings::PCI_VENDOR_ID_CIRRUS, // 0x1013 + IBM = bindings::PCI_VENDOR_ID_IBM, // 0x1014 + UNISYS = bindings::PCI_VENDOR_ID_UNISYS, // 0x1018 + COMPEX2 = bindings::PCI_VENDOR_ID_COMPEX2, // 0x101a + WD = bindings::PCI_VENDOR_ID_WD, // 0x101c + AMI = bindings::PCI_VENDOR_ID_AMI, // 0x101e + AMD = bindings::PCI_VENDOR_ID_AMD, // 0x1022 + TRIDENT = bindings::PCI_VENDOR_ID_TRIDENT, // 0x1023 + AI = bindings::PCI_VENDOR_ID_AI, // 0x1025 + DELL = bindings::PCI_VENDOR_ID_DELL, // 0x1028 + MATROX = bindings::PCI_VENDOR_ID_MATROX, // 0x102B + MOBILITY_ELECTRONICS = bindings::PCI_VENDOR_ID_MOBILITY_ELECTRONICS, // 0x14f2 + CT = bindings::PCI_VENDOR_ID_CT, // 0x102c + MIRO = bindings::PCI_VENDOR_ID_MIRO, // 0x1031 + NEC = bindings::PCI_VENDOR_ID_NEC, // 0x1033 + FD = bindings::PCI_VENDOR_ID_FD, // 0x1036 + SI = bindings::PCI_VENDOR_ID_SI, // 0x1039 + HP = bindings::PCI_VENDOR_ID_HP, // 0x103c + HP_3PAR = bindings::PCI_VENDOR_ID_HP_3PAR, // 0x1590 + PCTECH = bindings::PCI_VENDOR_ID_PCTECH, // 0x1042 + ASUSTEK = bindings::PCI_VENDOR_ID_ASUSTEK, // 0x1043 + DPT = bindings::PCI_VENDOR_ID_DPT, // 0x1044 + OPTI = bindings::PCI_VENDOR_ID_OPTI, // 0x1045 + ELSA = bindings::PCI_VENDOR_ID_ELSA, // 0x1048 + STMICRO = bindings::PCI_VENDOR_ID_STMICRO, // 0x104A + BUSLOGIC = bindings::PCI_VENDOR_ID_BUSLOGIC, // 0x104B + TI = bindings::PCI_VENDOR_ID_TI, // 0x104c + SONY = bindings::PCI_VENDOR_ID_SONY, // 0x104d + WINBOND2 = bindings::PCI_VENDOR_ID_WINBOND2, // 0x1050 + ANIGMA = bindings::PCI_VENDOR_ID_ANIGMA, // 0x1051 + EFAR = bindings::PCI_VENDOR_ID_EFAR, // 0x1055 + MOTOROLA = bindings::PCI_VENDOR_ID_MOTOROLA, // 0x1057 + PROMISE = bindings::PCI_VENDOR_ID_PROMISE, // 0x105a + FOXCONN = bindings::PCI_VENDOR_ID_FOXCONN, // 0x105b + UMC = bindings::PCI_VENDOR_ID_UMC, // 0x1060 + PICOPOWER = bindings::PCI_VENDOR_ID_PICOPOWER, // 0x1066 + MYLEX = bindings::PCI_VENDOR_ID_MYLEX, // 0x1069 + APPLE = bindings::PCI_VENDOR_ID_APPLE, // 0x106b + YAMAHA = bindings::PCI_VENDOR_ID_YAMAHA, // 0x1073 + QLOGIC = bindings::PCI_VENDOR_ID_QLOGIC, // 0x1077 + CYRIX = bindings::PCI_VENDOR_ID_CYRIX, // 0x1078 + CONTAQ = bindings::PCI_VENDOR_ID_CONTAQ, // 0x1080 + OLICOM = bindings::PCI_VENDOR_ID_OLICOM, // 0x108d + SUN = bindings::PCI_VENDOR_ID_SUN, // 0x108e + NI = bindings::PCI_VENDOR_ID_NI, // 0x1093 + CMD = bindings::PCI_VENDOR_ID_CMD, // 0x1095 + BROOKTREE = bindings::PCI_VENDOR_ID_BROOKTREE, // 0x109e + SGI = bindings::PCI_VENDOR_ID_SGI, // 0x10a9 + WINBOND = bindings::PCI_VENDOR_ID_WINBOND, // 0x10ad + PLX = bindings::PCI_VENDOR_ID_PLX, // 0x10b5 + MADGE = bindings::PCI_VENDOR_ID_MADGE, // 0x10b6 + THREECOM = bindings::PCI_VENDOR_ID_3COM, // 0x10b7 + AL = bindings::PCI_VENDOR_ID_AL, // 0x10b9 + NEOMAGIC = bindings::PCI_VENDOR_ID_NEOMAGIC, // 0x10c8 + TCONRAD = bindings::PCI_VENDOR_ID_TCONRAD, // 0x10da + ROHM = bindings::PCI_VENDOR_ID_ROHM, // 0x10db + NVIDIA = bindings::PCI_VENDOR_ID_NVIDIA, // 0x10de + IMS = bindings::PCI_VENDOR_ID_IMS, // 0x10e0 + AMCC = bindings::PCI_VENDOR_ID_AMCC, // 0x10e8 + AMPERE = bindings::PCI_VENDOR_ID_AMPERE, // 0x1def + INTERG = bindings::PCI_VENDOR_ID_INTERG, // 0x10ea + REALTEK = bindings::PCI_VENDOR_ID_REALTEK, // 0x10ec + XILINX = bindings::PCI_VENDOR_ID_XILINX, // 0x10ee + INIT = bindings::PCI_VENDOR_ID_INIT, // 0x1101 + CREATIVE = bindings::PCI_VENDOR_ID_CREATIVE, // 0x1102 + TTI = bindings::PCI_VENDOR_ID_TTI, // 0x1103 + SIGMA = bindings::PCI_VENDOR_ID_SIGMA, // 0x1105 + VIA = bindings::PCI_VENDOR_ID_VIA, // 0x1106 + SIEMENS = bindings::PCI_VENDOR_ID_SIEMENS, // 0x110A + VORTEX = bindings::PCI_VENDOR_ID_VORTEX, // 0x1119 + EF = bindings::PCI_VENDOR_ID_EF, // 0x111a + IDT = bindings::PCI_VENDOR_ID_IDT, // 0x111d + FORE = bindings::PCI_VENDOR_ID_FORE, // 0x1127 + PHILIPS = bindings::PCI_VENDOR_ID_PHILIPS, // 0x1131 + EICON = bindings::PCI_VENDOR_ID_EICON, // 0x1133 + CISCO = bindings::PCI_VENDOR_ID_CISCO, // 0x1137 + ZIATECH = bindings::PCI_VENDOR_ID_ZIATECH, // 0x1138 + SYSKONNECT = bindings::PCI_VENDOR_ID_SYSKONNECT, // 0x1148 + DIGI = bindings::PCI_VENDOR_ID_DIGI, // 0x114f + XIRCOM = bindings::PCI_VENDOR_ID_XIRCOM, // 0x115d + SERVERWORKS = bindings::PCI_VENDOR_ID_SERVERWORKS, // 0x1166 + ALTERA = bindings::PCI_VENDOR_ID_ALTERA, // 0x1172 + SBE = bindings::PCI_VENDOR_ID_SBE, // 0x1176 + TOSHIBA = bindings::PCI_VENDOR_ID_TOSHIBA, // 0x1179 + TOSHIBA_2 = bindings::PCI_VENDOR_ID_TOSHIBA_2, // 0x102f + ATTO = bindings::PCI_VENDOR_ID_ATTO, // 0x117c + RICOH = bindings::PCI_VENDOR_ID_RICOH, // 0x1180 + DLINK = bindings::PCI_VENDOR_ID_DLINK, // 0x1186 + ARTOP = bindings::PCI_VENDOR_ID_ARTOP, // 0x1191 + ZEITNET = bindings::PCI_VENDOR_ID_ZEITNET, // 0x1193 + FUJITSU_ME = bindings::PCI_VENDOR_ID_FUJITSU_ME, // 0x119e + MARVELL = bindings::PCI_VENDOR_ID_MARVELL, // 0x11ab + MARVELL_EXT = bindings::PCI_VENDOR_ID_MARVELL_EXT, // 0x1b4b + V3 = bindings::PCI_VENDOR_ID_V3, // 0x11b0 + ATT = bindings::PCI_VENDOR_ID_ATT, // 0x11c1 + SPECIALIX = bindings::PCI_VENDOR_ID_SPECIALIX, // 0x11cb + ANALOG_DEVICES = bindings::PCI_VENDOR_ID_ANALOG_DEVICES, // 0x11d4 + ZORAN = bindings::PCI_VENDOR_ID_ZORAN, // 0x11de + COMPEX = bindings::PCI_VENDOR_ID_COMPEX, // 0x11f6 + MICROSEMI = bindings::PCI_VENDOR_ID_MICROSEMI, // 0x11f8 + RP = bindings::PCI_VENDOR_ID_RP, // 0x11fe + CYCLADES = bindings::PCI_VENDOR_ID_CYCLADES, // 0x120e + O2 = bindings::PCI_VENDOR_ID_O2, // 0x1217 + THREEDX = bindings::PCI_VENDOR_ID_3DFX, // 0x121a + AVM = bindings::PCI_VENDOR_ID_AVM, // 0x1244 + STALLION = bindings::PCI_VENDOR_ID_STALLION, // 0x124d + AT = bindings::PCI_VENDOR_ID_AT, // 0x1259 + ASIX = bindings::PCI_VENDOR_ID_ASIX, // 0x125b + ESS = bindings::PCI_VENDOR_ID_ESS, // 0x125d + SATSAGEM = bindings::PCI_VENDOR_ID_SATSAGEM, // 0x1267 + ENSONIQ = bindings::PCI_VENDOR_ID_ENSONIQ, // 0x1274 + TRANSMETA = bindings::PCI_VENDOR_ID_TRANSMETA, // 0x1279 + ROCKWELL = bindings::PCI_VENDOR_ID_ROCKWELL, // 0x127A + ITE = bindings::PCI_VENDOR_ID_ITE, // 0x1283 + ALTEON = bindings::PCI_VENDOR_ID_ALTEON, // 0x12ae + NVIDIA_SGS = bindings::PCI_VENDOR_ID_NVIDIA_SGS, // 0x12d2 + PERICOM = bindings::PCI_VENDOR_ID_PERICOM, // 0x12D8 + AUREAL = bindings::PCI_VENDOR_ID_AUREAL, // 0x12eb + ELECTRONICDESIGNGMBH = bindings::PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, // 0x12f8 + ESDGMBH = bindings::PCI_VENDOR_ID_ESDGMBH, // 0x12fe + CB = bindings::PCI_VENDOR_ID_CB, // 0x1307 + SIIG = bindings::PCI_VENDOR_ID_SIIG, // 0x131f + RADISYS = bindings::PCI_VENDOR_ID_RADISYS, // 0x1331 + MICRO_MEMORY = bindings::PCI_VENDOR_ID_MICRO_MEMORY, // 0x1332 + DOMEX = bindings::PCI_VENDOR_ID_DOMEX, // 0x134a + INTASHIELD = bindings::PCI_VENDOR_ID_INTASHIELD, // 0x135a + QUATECH = bindings::PCI_VENDOR_ID_QUATECH, // 0x135C + SEALEVEL = bindings::PCI_VENDOR_ID_SEALEVEL, // 0x135e + HYPERCOPE = bindings::PCI_VENDOR_ID_HYPERCOPE, // 0x1365 + DIGIGRAM = bindings::PCI_VENDOR_ID_DIGIGRAM, // 0x1369 + KAWASAKI = bindings::PCI_VENDOR_ID_KAWASAKI, // 0x136b + CNET = bindings::PCI_VENDOR_ID_CNET, // 0x1371 + LMC = bindings::PCI_VENDOR_ID_LMC, // 0x1376 + NETGEAR = bindings::PCI_VENDOR_ID_NETGEAR, // 0x1385 + APPLICOM = bindings::PCI_VENDOR_ID_APPLICOM, // 0x1389 + MOXA = bindings::PCI_VENDOR_ID_MOXA, // 0x1393 + CCD = bindings::PCI_VENDOR_ID_CCD, // 0x1397 + EXAR = bindings::PCI_VENDOR_ID_EXAR, // 0x13a8 + MICROGATE = bindings::PCI_VENDOR_ID_MICROGATE, // 0x13c0 + THREEWARE = bindings::PCI_VENDOR_ID_3WARE, // 0x13C1 + IOMEGA = bindings::PCI_VENDOR_ID_IOMEGA, // 0x13ca + ABOCOM = bindings::PCI_VENDOR_ID_ABOCOM, // 0x13D1 + SUNDANCE = bindings::PCI_VENDOR_ID_SUNDANCE, // 0x13f0 + CMEDIA = bindings::PCI_VENDOR_ID_CMEDIA, // 0x13f6 + ADVANTECH = bindings::PCI_VENDOR_ID_ADVANTECH, // 0x13fe + MEILHAUS = bindings::PCI_VENDOR_ID_MEILHAUS, // 0x1402 + LAVA = bindings::PCI_VENDOR_ID_LAVA, // 0x1407 + TIMEDIA = bindings::PCI_VENDOR_ID_TIMEDIA, // 0x1409 + ICE = bindings::PCI_VENDOR_ID_ICE, // 0x1412 + MICROSOFT = bindings::PCI_VENDOR_ID_MICROSOFT, // 0x1414 + OXSEMI = bindings::PCI_VENDOR_ID_OXSEMI, // 0x1415 + CHELSIO = bindings::PCI_VENDOR_ID_CHELSIO, // 0x1425 + EDIMAX = bindings::PCI_VENDOR_ID_EDIMAX, // 0x1432 + ADLINK = bindings::PCI_VENDOR_ID_ADLINK, // 0x144a + SAMSUNG = bindings::PCI_VENDOR_ID_SAMSUNG, // 0x144d + GIGABYTE = bindings::PCI_VENDOR_ID_GIGABYTE, // 0x1458 + AMBIT = bindings::PCI_VENDOR_ID_AMBIT, // 0x1468 + MYRICOM = bindings::PCI_VENDOR_ID_MYRICOM, // 0x14c1 + MEDIATEK = bindings::PCI_VENDOR_ID_MEDIATEK, // 0x14c3 + TITAN = bindings::PCI_VENDOR_ID_TITAN, // 0x14D2 + PANACOM = bindings::PCI_VENDOR_ID_PANACOM, // 0x14d4 + SIPACKETS = bindings::PCI_VENDOR_ID_SIPACKETS, // 0x14d9 + AFAVLAB = bindings::PCI_VENDOR_ID_AFAVLAB, // 0x14db + AMPLICON = bindings::PCI_VENDOR_ID_AMPLICON, // 0x14dc + BCM_GVC = bindings::PCI_VENDOR_ID_BCM_GVC, // 0x14a4 + BROADCOM = bindings::PCI_VENDOR_ID_BROADCOM, // 0x14e4 + TOPIC = bindings::PCI_VENDOR_ID_TOPIC, // 0x151f + MAINPINE = bindings::PCI_VENDOR_ID_MAINPINE, // 0x1522 + ENE = bindings::PCI_VENDOR_ID_ENE, // 0x1524 + SYBA = bindings::PCI_VENDOR_ID_SYBA, // 0x1592 + MORETON = bindings::PCI_VENDOR_ID_MORETON, // 0x15aa + VMWARE = bindings::PCI_VENDOR_ID_VMWARE, // 0x15ad + ZOLTRIX = bindings::PCI_VENDOR_ID_ZOLTRIX, // 0x15b0 + MELLANOX = bindings::PCI_VENDOR_ID_MELLANOX, // 0x15b3 + DFI = bindings::PCI_VENDOR_ID_DFI, // 0x15bd + QUICKNET = bindings::PCI_VENDOR_ID_QUICKNET, // 0x15e2 + ADDIDATA = bindings::PCI_VENDOR_ID_ADDIDATA, // 0x15B8 + PDC = bindings::PCI_VENDOR_ID_PDC, // 0x15e9 + FARSITE = bindings::PCI_VENDOR_ID_FARSITE, // 0x1619 + ARIMA = bindings::PCI_VENDOR_ID_ARIMA, // 0x161f + BROCADE = bindings::PCI_VENDOR_ID_BROCADE, // 0x1657 + SIBYTE = bindings::PCI_VENDOR_ID_SIBYTE, // 0x166d + ATHEROS = bindings::PCI_VENDOR_ID_ATHEROS, // 0x168c + NETCELL = bindings::PCI_VENDOR_ID_NETCELL, // 0x169c + CENATEK = bindings::PCI_VENDOR_ID_CENATEK, // 0x16CA + SYNOPSYS = bindings::PCI_VENDOR_ID_SYNOPSYS, // 0x16c3 + USR = bindings::PCI_VENDOR_ID_USR, // 0x16ec + VITESSE = bindings::PCI_VENDOR_ID_VITESSE, // 0x1725 + LINKSYS = bindings::PCI_VENDOR_ID_LINKSYS, // 0x1737 + ALTIMA = bindings::PCI_VENDOR_ID_ALTIMA, // 0x173b + CAVIUM = bindings::PCI_VENDOR_ID_CAVIUM, // 0x177d + TECHWELL = bindings::PCI_VENDOR_ID_TECHWELL, // 0x1797 + BELKIN = bindings::PCI_VENDOR_ID_BELKIN, // 0x1799 + RDC = bindings::PCI_VENDOR_ID_RDC, // 0x17f3 + GLI = bindings::PCI_VENDOR_ID_GLI, // 0x17a0 + LENOVO = bindings::PCI_VENDOR_ID_LENOVO, // 0x17aa + QCOM = bindings::PCI_VENDOR_ID_QCOM, // 0x17cb + CDNS = bindings::PCI_VENDOR_ID_CDNS, // 0x17cd + ARECA = bindings::PCI_VENDOR_ID_ARECA, // 0x17d3 + S2IO = bindings::PCI_VENDOR_ID_S2IO, // 0x17d5 + SITECOM = bindings::PCI_VENDOR_ID_SITECOM, // 0x182d + TOPSPIN = bindings::PCI_VENDOR_ID_TOPSPIN, // 0x1867 + COMMTECH = bindings::PCI_VENDOR_ID_COMMTECH, // 0x18f7 + SILAN = bindings::PCI_VENDOR_ID_SILAN, // 0x1904 + RENESAS = bindings::PCI_VENDOR_ID_RENESAS, // 0x1912 + SOLARFLARE = bindings::PCI_VENDOR_ID_SOLARFLARE, // 0x1924 + TDI = bindings::PCI_VENDOR_ID_TDI, // 0x192E + NXP = bindings::PCI_VENDOR_ID_NXP, // 0x1957 + PASEMI = bindings::PCI_VENDOR_ID_PASEMI, // 0x1959 + ATTANSIC = bindings::PCI_VENDOR_ID_ATTANSIC, // 0x1969 + JMICRON = bindings::PCI_VENDOR_ID_JMICRON, // 0x197B + KORENIX = bindings::PCI_VENDOR_ID_KORENIX, // 0x1982 + HUAWEI = bindings::PCI_VENDOR_ID_HUAWEI, // 0x19e5 + NETRONOME = bindings::PCI_VENDOR_ID_NETRONOME, // 0x19ee + QMI = bindings::PCI_VENDOR_ID_QMI, // 0x1a32 + AZWAVE = bindings::PCI_VENDOR_ID_AZWAVE, // 0x1a3b + REDHAT_QUMRANET = bindings::PCI_VENDOR_ID_REDHAT_QUMRANET, // 0x1af4 + ASMEDIA = bindings::PCI_VENDOR_ID_ASMEDIA, // 0x1b21 + REDHAT = bindings::PCI_VENDOR_ID_REDHAT, // 0x1b36 + WCHIC = bindings::PCI_VENDOR_ID_WCHIC, // 0x1c00 + SILICOM_DENMARK = bindings::PCI_VENDOR_ID_SILICOM_DENMARK, // 0x1c2c + AMAZON_ANNAPURNA_LABS = bindings::PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, // 0x1c36 + CIRCUITCO = bindings::PCI_VENDOR_ID_CIRCUITCO, // 0x1cc8 + AMAZON = bindings::PCI_VENDOR_ID_AMAZON, // 0x1d0f + ZHAOXIN = bindings::PCI_VENDOR_ID_ZHAOXIN, // 0x1d17 + ROCKCHIP = bindings::PCI_VENDOR_ID_ROCKCHIP, // 0x1d87 + HYGON = bindings::PCI_VENDOR_ID_HYGON, // 0x1d94 + META = bindings::PCI_VENDOR_ID_META, // 0x1d9b + FUNGIBLE = bindings::PCI_VENDOR_ID_FUNGIBLE, // 0x1dad + HXT = bindings::PCI_VENDOR_ID_HXT, // 0x1dbf + TEKRAM = bindings::PCI_VENDOR_ID_TEKRAM, // 0x1de1 + RPI = bindings::PCI_VENDOR_ID_RPI, // 0x1de4 + ALIBABA = bindings::PCI_VENDOR_ID_ALIBABA, // 0x1ded + CXL = bindings::PCI_VENDOR_ID_CXL, // 0x1e98 + TEHUTI = bindings::PCI_VENDOR_ID_TEHUTI, // 0x1fc9 + SUNIX = bindings::PCI_VENDOR_ID_SUNIX, // 0x1fd4 + HINT = bindings::PCI_VENDOR_ID_HINT, // 0x3388 + THREEDLABS = bindings::PCI_VENDOR_ID_3DLABS, // 0x3d3d + NETXEN = bindings::PCI_VENDOR_ID_NETXEN, // 0x4040 + AKS = bindings::PCI_VENDOR_ID_AKS, // 0x416c + WCHCN = bindings::PCI_VENDOR_ID_WCHCN, // 0x4348 + ACCESSIO = bindings::PCI_VENDOR_ID_ACCESSIO, // 0x494f + S3 = bindings::PCI_VENDOR_ID_S3, // 0x5333 + DUNORD = bindings::PCI_VENDOR_ID_DUNORD, // 0x5544 + DCI = bindings::PCI_VENDOR_ID_DCI, // 0x6666 + GLENFLY = bindings::PCI_VENDOR_ID_GLENFLY, // 0x6766 + INTEL = bindings::PCI_VENDOR_ID_INTEL, // 0x8086 + WANGXUN = bindings::PCI_VENDOR_ID_WANGXUN, // 0x8088 + SCALEMP = bindings::PCI_VENDOR_ID_SCALEMP, // 0x8686 + COMPUTONE = bindings::PCI_VENDOR_ID_COMPUTONE, // 0x8e0e + KTI = bindings::PCI_VENDOR_ID_KTI, // 0x8e2e + ADAPTEC = bindings::PCI_VENDOR_ID_ADAPTEC, // 0x9004 + ADAPTEC2 = bindings::PCI_VENDOR_ID_ADAPTEC2, // 0x9005 + HOLTEK = bindings::PCI_VENDOR_ID_HOLTEK, // 0x9412 + NETMOS = bindings::PCI_VENDOR_ID_NETMOS, // 0x9710 + THREECOM_2 = bindings::PCI_VENDOR_ID_3COM_2, // 0xa727 + SOLIDRUN = bindings::PCI_VENDOR_ID_SOLIDRUN, // 0xd063 + DIGIUM = bindings::PCI_VENDOR_ID_DIGIUM, // 0xd161 + TIGERJET = bindings::PCI_VENDOR_ID_TIGERJET, // 0xe159 + XILINX_RME = bindings::PCI_VENDOR_ID_XILINX_RME, // 0xea60 + XEN = bindings::PCI_VENDOR_ID_XEN, // 0x5853 + OCZ = bindings::PCI_VENDOR_ID_OCZ, // 0x1b85 + NCUBE = bindings::PCI_VENDOR_ID_NCUBE, // 0x10ff +} diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs new file mode 100644 index 000000000..953e16735 --- /dev/null +++ b/rust/kernel/pci/io.rs @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI memory-mapped I/O infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + devres::DevresLt, + io::{ + IoBackend, + IoBase, + IoCapable, + Mmio, + MmioBackend, + MmioRaw, + Region, // + }, + prelude::*, + ptr::KnownSize, + types::{ + CovariantForLt, + ForLt, // + }, // +}; + +/// Represents the size of a PCI configuration space. +/// +/// PCI devices can have either a *normal* (legacy) configuration space of 256 bytes, +/// or an *extended* configuration space of 4096 bytes as defined in the PCI Express +/// specification. +#[repr(usize)] +#[derive(Eq, PartialEq)] +pub enum ConfigSpaceSize { + /// 256-byte legacy PCI configuration space. + Normal = 256, + + /// 4096-byte PCIe extended configuration space. + Extended = 4096, +} + +impl ConfigSpaceSize { + /// Get the raw value of this enum. + #[inline(always)] + pub const fn into_raw(self) -> usize { + // CAST: PCI configuration space size is at most 4096 bytes, so the value always fits + // within `usize` without truncation or sign change. + self as usize + } +} + +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; + +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; + +/// A view of PCI configuration space of a device. +/// +/// Provides typed read and write accessors for configuration registers +/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. +/// +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). +/// +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { + pub(crate) pdev: &'a Device, + ptr: *mut T, +} + +impl Copy for ConfigSpace<'_, T> {} +impl Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for ConfigSpace<'_, T> {} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } +} + +/// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_config_space_io_capable { + ($ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + + let mut val: $ty = 0; + + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; + val + } + + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; + } + } + }; +} + +// PCI configuration space supports 8, 16, and 32-bit accesses. +impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); +impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); +impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ConfigSpace<'a, T> { + self + } +} + +/// A PCI BAR to perform I/O-Operations on. +/// +/// I/O backend assumes that the device is little-endian and will automatically +/// convert from little-endian to CPU endianness. +/// +/// # Invariants +/// +/// `Bar` always holds an `IoRaw` instance that holds a valid pointer to the start of the I/O +/// memory mapped PCI BAR and its size. +pub struct Bar<'a, const SIZE: usize = 0> { + pdev: &'a Device, + io: MmioRaw>, + num: i32, +} + +impl ForLt for Bar<'static, SIZE> { + type Of<'a> = Bar<'a, SIZE>; +} + +// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for Bar<'static, SIZE> {} + +/// A device-managed PCI BAR mapping. +/// +/// See [`Bar::into_devres`]. +pub type DevresBar = DevresLt>; + +impl<'a, const SIZE: usize> Bar<'a, SIZE> { + pub(super) fn new( + pdev: &'a Device, + num: u32, + name: &'static CStr, + ) -> Result { + let len = pdev.resource_len(num)?; + if len == 0 { + return Err(ENOMEM); + } + + // Convert to `i32`, since that's what all the C bindings use. + let num = i32::try_from(num)?; + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) }; + if ret != 0 { + return Err(EBUSY); + } + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize; + if ioptr == 0 { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { bindings::pci_release_region(pdev.as_raw(), num) }; + return Err(ENOMEM); + } + + let io = match MmioRaw::new_region(ioptr, len as usize) { + Ok(io) => io, + Err(err) => { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { Self::do_release(pdev, ioptr, num) }; + return Err(err); + } + }; + + Ok(Bar { pdev, io, num }) + } + + /// # Safety + /// + /// `ioptr` must be a valid pointer to the memory mapped PCI BAR number `num`. + unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is valid by the safety requirements. + // `num` is valid by the safety requirements. + unsafe { + bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut c_void); + bindings::pci_release_region(pdev.as_raw(), num); + } + } + + fn release(&self) { + // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. + unsafe { Self::do_release(self.pdev, self.io.addr(), self.num) }; + } + + /// Consume the `Bar` and register it as a device-managed resource. + /// + /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data. + /// Access to the BAR is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let pdev = self.pdev; + // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(pdev.as_ref(), self) } + } +} + +impl Bar<'_> { + #[inline] + pub(super) fn index_is_valid(index: u32) -> bool { + // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. + index < bindings::PCI_NUM_RESOURCES + } +} + +impl Drop for Bar<'_, SIZE> { + fn drop(&mut self) { + self.release(); + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. + unsafe { Mmio::from_raw(self.io) } + } +} + +impl Device { + /// Maps an entire PCI BAR after performing a region-request on it. I/O operation bound checks + /// can be performed on compile time for offsets (plus the requested type size) < SIZE. + pub fn iomap_region_sized<'a, const SIZE: usize>( + &'a self, + bar: u32, + name: &'static CStr, + ) -> Result> { + Bar::new(self, bar, name) + } + + /// Maps an entire PCI BAR after performing a region-request on it. + pub fn iomap_region<'a>(&'a self, bar: u32, name: &'static CStr) -> Result> { + self.iomap_region_sized::<0>(bar, name) + } + + /// Returns the size of configuration space. + pub fn cfg_size(&self) -> ConfigSpaceSize { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + let size = unsafe { (*self.as_raw()).cfg_size }; + match size { + 256 => ConfigSpaceSize::Normal, + 4096 => ConfigSpaceSize::Extended, + _ => { + // PANIC: The PCI subsystem only ever reports the configuration space size as either + // `ConfigSpaceSize::Normal` or `ConfigSpaceSize::Extended`. + unreachable!(); + } + } + } + + /// Return a view of the normal (256-byte) config space. + pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. + ConfigSpace { + pdev: self, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), + } + } + + /// Return a view of the extended (4096-byte) config space. + pub fn config_space_extended<'a>(&'a self) -> Result> { + if self.cfg_size() != ConfigSpaceSize::Extended { + return Err(EINVAL); + } + + // INVARIANT: null is aligned and we just checked the `cfg_size`. + Ok(ConfigSpace { + pdev: self, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), + }) + } +} diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs new file mode 100644 index 000000000..22e2cdf82 --- /dev/null +++ b/rust/kernel/pci/irq.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI interrupt infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + device::Bound, + error::to_result, + irq::IrqRequest, + prelude::*, // +}; +use core::num::NonZero; + +/// IRQ type flags for PCI interrupt allocation. +#[derive(Debug, Clone, Copy)] +pub enum IrqType { + /// INTx interrupts. + Intx, + /// Message Signaled Interrupts (MSI). + Msi, + /// Extended Message Signaled Interrupts (MSI-X). + MsiX, +} + +impl IrqType { + /// Convert to the corresponding kernel flags. + const fn as_raw(self) -> u32 { + match self { + IrqType::Intx => bindings::PCI_IRQ_INTX, + IrqType::Msi => bindings::PCI_IRQ_MSI, + IrqType::MsiX => bindings::PCI_IRQ_MSIX, + } + } + + /// Construct from raw value. + #[inline] + const fn from_raw(raw: u32) -> Self { + match raw { + bindings::PCI_IRQ_MSIX => IrqType::MsiX, + bindings::PCI_IRQ_MSI => IrqType::Msi, + _ => IrqType::Intx, + } + } +} + +/// Set of IRQ types that can be used for PCI interrupt allocation. +#[derive(Debug, Clone, Copy, Default)] +pub struct IrqTypes(u32); + +impl IrqTypes { + /// Create a set containing all IRQ types (MSI-X, MSI, and INTx). + pub const fn all() -> Self { + Self(bindings::PCI_IRQ_ALL_TYPES) + } + + /// Build a set of IRQ types. + /// + /// # Examples + /// + /// ```ignore + /// // Create a set with only MSI and MSI-X (no INTx interrupts). + /// let msi_only = IrqTypes::default() + /// .with(IrqType::Msi) + /// .with(IrqType::MsiX); + /// ``` + pub const fn with(self, irq_type: IrqType) -> Self { + Self(self.0 | irq_type.as_raw()) + } + + /// Get the raw flags value. + const fn as_raw(self) -> u32 { + self.0 + } +} + +/// A resolved IRQ vector from a PCI interrupt vector allocation. +/// +/// Created by [`IrqVectorRegistration::index`]. Convert to [`IrqRequest`] via [`From`] to register +/// a handler with [`irq::Registration::new`](crate::irq::Registration::new). +pub struct IrqVector<'a> { + request: IrqRequest<'a>, + reg: &'a IrqVectorRegistration<'a>, +} + +impl<'a> IrqVector<'a> { + /// Creates a new [`IrqVector`] with an already resolved [`IrqRequest`]. + /// + /// # Safety + /// + /// `request` must have been resolved from `reg`. + #[inline] + unsafe fn new(request: IrqRequest<'a>, reg: &'a IrqVectorRegistration<'a>) -> Self { + Self { request, reg } + } + + /// Returns the [`IrqVectorRegistration`] this vector was derived from. + #[inline] + pub fn vectors(&self) -> &'a IrqVectorRegistration<'a> { + self.reg + } + + /// Returns the interrupt type the PCI core selected for this vector's allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + self.reg.irq_type() + } +} + +impl<'a> From> for IrqRequest<'a> { + #[inline] + fn from(vector: IrqVector<'a>) -> Self { + vector.request + } +} + +/// An allocation of PCI interrupt vectors for a device. +/// +/// This type owns the vector allocation; dropping it frees the vectors. IRQ handlers borrow from +/// this registration and must be dropped before it is. +/// +/// # Invariants +/// +/// `dev` has an allocation of `len` interrupt vectors. +pub struct IrqVectorRegistration<'a> { + dev: &'a Device, + len: NonZero, +} + +impl<'a> IrqVectorRegistration<'a> { + /// Returns the number of allocated vectors. + /// + /// This is at least the `min_vecs` that [`Device::alloc_irq_vectors`] was asked for. + #[inline] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len.get() + } + + /// Returns the interrupt type the PCI core selected for this allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + IrqType::from_raw(unsafe { bindings::pci_irq_type(self.dev.as_raw()) }) + } + + /// Returns the [`IrqVector`] at `index`. + /// + /// Returns [`EINVAL`] if the `index` is out of bounds for the length reported by + /// [`Self::len()`]. + #[inline] + pub fn index(&self, index: usize) -> Result> { + let index = u32::try_from(index)?; + + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index) }; + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is a valid IRQ number for `self.dev`, resolved from this registration. + Ok(unsafe { IrqVector::new(IrqRequest::new(self.dev.as_ref(), irq as u32), self) }) + } +} + +impl Drop for IrqVectorRegistration<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.dev.as_raw()` is a valid pointer to a + // `struct pci_dev` that has successfully allocated IRQ vectors. + unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; + } +} + +impl Device { + /// Allocate IRQ vectors for this PCI device. + /// + /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. + /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` + /// parameter and hardware capabilities. When multiple types are specified, the kernel + /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. + /// + /// The allocated vectors are freed when the returned [`IrqVectorRegistration`] is dropped. + /// Use [`IrqVectorRegistration::index`] to obtain an [`IrqVector`] for a given vector + /// index. + /// + /// # Arguments + /// + /// * `min_vecs` - Minimum number of vectors required. + /// * `max_vecs` - Maximum number of vectors to allocate. + /// * `irq_types` - Types of interrupts that can be used. + /// + /// # Returns + /// + /// Returns the IRQ vector registration, or an error if `min_vecs` vectors cannot be + /// allocated. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{ device::Bound, pci}; + /// # fn no_run(dev: &pci::Device) -> Result { + /// // Allocate using any available interrupt type in the order mentioned above. + /// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?; + /// + /// // Allocate MSI or MSI-X only (no INTx interrupts). + /// let msi_only = pci::IrqTypes::default() + /// .with(pci::IrqType::Msi) + /// .with(pci::IrqType::MsiX); + /// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?; + /// # Ok(()) + /// # } + /// ``` + pub fn alloc_irq_vectors( + &self, + min_vecs: u32, + max_vecs: u32, + irq_types: IrqTypes, + ) -> Result> { + // SAFETY: + // - `self.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + to_result(ret)?; + + let len = NonZero::new(ret as usize).ok_or(EINVAL)?; + + // INVARIANT: `pci_alloc_irq_vectors()` allocated `len` vectors for `self`. + Ok(IrqVectorRegistration { dev: self, len }) + } +} diff --git a/rust/kernel/pid_namespace.rs b/rust/kernel/pid_namespace.rs new file mode 100644 index 000000000..979a9718f --- /dev/null +++ b/rust/kernel/pid_namespace.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (c) 2024 Christian Brauner + +//! Pid namespaces. +//! +//! C header: [`include/linux/pid_namespace.h`](srctree/include/linux/pid_namespace.h) and +//! [`include/linux/pid.h`](srctree/include/linux/pid.h) + +use crate::{bindings, sync::aref::AlwaysRefCounted, types::Opaque}; +use core::ptr; + +/// Wraps the kernel's `struct pid_namespace`. Thread safe. +/// +/// This structure represents the Rust abstraction for a C `struct pid_namespace`. This +/// implementation abstracts the usage of an already existing C `struct pid_namespace` within Rust +/// code that we get passed from the C side. +#[repr(transparent)] +pub struct PidNamespace { + inner: Opaque, +} + +impl PidNamespace { + /// Returns a raw pointer to the inner C struct. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::pid_namespace { + self.inner.get() + } + + /// Creates a reference to a [`PidNamespace`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`PidNamespace`] reference. + pub unsafe fn from_ptr<'a>(ptr: *const bindings::pid_namespace) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `PidNamespace` type being transparent makes the cast ok. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: Instances of `PidNamespace` are always reference-counted. +unsafe impl AlwaysRefCounted for PidNamespace { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_pid_ns(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_pid_ns(obj.cast().as_ptr()) } + } +} + +// SAFETY: +// - `PidNamespace::dec_ref` can be called from any thread. +// - It is okay to send ownership of `PidNamespace` across thread boundaries. +unsafe impl Send for PidNamespace {} + +// SAFETY: It's OK to access `PidNamespace` through shared references from other threads because +// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for PidNamespace {} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs new file mode 100644 index 000000000..ac0a012ae --- /dev/null +++ b/rust/kernel/platform.rs @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the platform bus. +//! +//! C header: [`include/linux/platform_device.h`](srctree/include/linux/platform_device.h) + +use crate::{ + acpi, + bindings, + container_of, + device::{ + self, + Bound, // + }, + driver, + error::{ + from_result, + to_result, // + }, + io::Resource, + irq::{ + self, + IrqRequest, // + }, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; + +#[cfg(CONFIG_HAS_IOMEM)] +use crate::io::mem::IoRequest; + +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +/// An adapter for the registration of platform drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::platform_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct platform_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::platform_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + pdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct platform_driver` on initialization. + unsafe { + (*pdrv.get()).driver.name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).driver.of_match_table = of_table; + (*pdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__platform_driver_register(pdrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(pdrv: &Opaque) { + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::platform_driver_unregister(pdrv.get()) }; + } +} + +impl Adapter { + extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int { + // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + // SAFETY: `pdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(pdev.as_ref()) }; + + from_result(|| { + let data = T::probe(pdev, info); + + pdev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { + // SAFETY: The platform bus only ever calls the remove callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; + + T::unbind(pdev, data); + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single platform driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_platform_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_platform_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::platform::Adapter, { $($f)* }); + }; +} + +/// The platform driver trait. +/// +/// Drivers must implement this trait in order to get a platform driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// # acpi, +/// # bindings, +/// # device::Core, +/// # of, +/// # platform, +/// # }; +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// impl platform::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// _pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// } +///``` +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// Platform driver probe. + /// + /// Called when a new platform device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + dev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The platform device representation. +/// +/// This structure represents the Rust abstraction for a C `struct platform_device`. The +/// implementation abstracts the usage of an already existing C `struct platform_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct platform_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::platform_device { + self.0.get() + } + + /// Returns the resource at `index`, if any. + pub fn resource_by_index(&self, index: u32) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct platform_device`. + let resource = unsafe { + bindings::platform_get_resource(self.as_raw(), bindings::IORESOURCE_MEM, index) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } + + /// Returns the resource with a given `name`, if any. + pub fn resource_by_name(&self, name: &CStr) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct + // platform_device` and `name` points to a valid C string. + let resource = unsafe { + bindings::platform_get_resource_byname( + self.as_raw(), + bindings::IORESOURCE_MEM, + name.as_char_ptr(), + ) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } +} + +#[cfg(CONFIG_HAS_IOMEM)] +impl Device { + /// Returns an `IoRequest` for the resource at `index`, if any. + pub fn io_request_by_index(&self, index: u32) -> Option> { + self.resource_by_index(index) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } + + /// Returns an `IoRequest` for the resource with a given `name`, if any. + pub fn io_request_by_name(&self, name: &CStr) -> Option> { + self.resource_by_name(name) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } +} + +// SAFETY: `platform::Device` is a transparent wrapper of `struct platform_device`. +// The offset is guaranteed to point to a valid device field inside `platform::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::platform_device, dev); +} + +macro_rules! define_irq_accessor_by_index { + ( + $(#[$meta:meta])* $fn_name:ident, + $request_fn:ident, + $reg_type:ident, + $handler_trait:ident + ) => { + $(#[$meta])* + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( + &'a self, + flags: irq::Flags, + index: u32, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(index)?; + + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) + }) + } + }; +} + +macro_rules! define_irq_accessor_by_name { + ( + $(#[$meta:meta])* $fn_name:ident, + $request_fn:ident, + $reg_type:ident, + $handler_trait:ident + ) => { + $(#[$meta])* + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( + &'a self, + flags: irq::Flags, + irq_name: &'a CStr, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(irq_name)?; + + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) + }) + } + }; +} + +impl Device { + /// Returns an [`IrqRequest`] for the IRQ at the given index, if any. + pub fn irq_by_index(&self, index: u32) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq(self.as_raw(), index) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ at the given index, but does not + /// print an error if the IRQ cannot be obtained. + pub fn optional_irq_by_index(&self, index: u32) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq_optional(self.as_raw(), index) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ with the given name, if any. + pub fn irq_by_name(&self, name: &CStr) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq_byname(self.as_raw(), name.as_char_ptr()) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ with the given name, but does not + /// print an error if the IRQ cannot be obtained. + pub fn optional_irq_by_name(&self, name: &CStr) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { + bindings::platform_get_irq_byname_optional(self.as_raw(), name.as_char_ptr()) + }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + define_irq_accessor_by_index!( + /// Returns a [`irq::Registration`] for the IRQ at the given index. + request_irq_by_index, + irq_by_index, + Registration, + Handler + ); + define_irq_accessor_by_name!( + /// Returns a [`irq::Registration`] for the IRQ with the given name. + request_irq_by_name, + irq_by_name, + Registration, + Handler + ); + define_irq_accessor_by_index!( + /// Does the same as [`Self::request_irq_by_index`], except that it does + /// not print an error message if the IRQ cannot be obtained. + request_optional_irq_by_index, + optional_irq_by_index, + Registration, + Handler + ); + define_irq_accessor_by_name!( + /// Does the same as [`Self::request_irq_by_name`], except that it does + /// not print an error message if the IRQ cannot be obtained. + request_optional_irq_by_name, + optional_irq_by_name, + Registration, + Handler + ); + + define_irq_accessor_by_index!( + /// Returns a [`irq::ThreadedRegistration`] for the IRQ at the given index. + request_threaded_irq_by_index, + irq_by_index, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_name!( + /// Returns a [`irq::ThreadedRegistration`] for the IRQ with the given name. + request_threaded_irq_by_name, + irq_by_name, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_index!( + /// Does the same as [`Self::request_threaded_irq_by_index`], except + /// that it does not print an error message if the IRQ cannot be + /// obtained. + request_optional_threaded_irq_by_index, + optional_irq_by_index, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_name!( + /// Does the same as [`Self::request_threaded_irq_by_name`], except that + /// it does not print an error message if the IRQ cannot be obtained. + request_optional_threaded_irq_by_name, + optional_irq_by_name, + ThreadedRegistration, + ThreadedHandler + ); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +impl<'a> crate::dma::Device<'a> for Device> {} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::platform_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct platform_device`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &Device { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if !unsafe { bindings::dev_is_platform(dev.as_raw()) } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the bus type of `dev` equals + // `bindings::platform_bus_type`, hence `dev` must be embedded in a valid + // `struct platform_device` as guaranteed by the corresponding C code. + let pdev = unsafe { container_of!(dev.as_raw(), bindings::platform_device, dev) }; + + // SAFETY: `pdev` is a valid pointer to a `struct platform_device`. + Ok(unsafe { &*pdev.cast() }) + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct platform_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs new file mode 100644 index 000000000..ca396f1f7 --- /dev/null +++ b/rust/kernel/prelude.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` prelude. +//! +//! These are the most common items used by Rust code in the kernel, +//! intended to be imported by all Rust code, for convenience. +//! +//! # Examples +//! +//! ``` +//! use kernel::prelude::*; +//! ``` + +#[doc(no_inline)] +pub use core::{ + mem::{ + align_of, + align_of_val, + size_of, + size_of_val, // + }, + pin::Pin, // +}; + +#[doc(no_inline)] +pub use ::ffi::{ + c_char, + c_int, + c_long, + c_longlong, + c_schar, + c_short, + c_uchar, + c_uint, + c_ulong, + c_ulonglong, + c_ushort, + c_void, + CStr, // +}; + +#[doc(no_inline)] +pub use macros::{ + export, + fmt, + kunit_tests, + module, + vtable, // +}; + +#[doc(no_inline)] +pub use pin_init::{ + init, + pin_data, + pin_init, + pinned_drop, + InPlaceWrite, + Init, + PinInit, + Zeroable, // +}; + +#[doc(no_inline)] +pub use zerocopy::{ + FromBytes, + IntoBytes, // +}; + +#[doc(no_inline)] +pub use zerocopy_derive::{ + FromBytes, + IntoBytes, // +}; + +#[doc(no_inline)] +pub use super::{ + alloc::{ + flags::*, + Box, + KBox, + KVBox, + KVVec, + KVec, + VBox, + VVec, + Vec, // + }, + build_assert::{ + build_assert, + build_error, + const_assert, + static_assert, // + }, + current, + dev_alert, + dev_crit, + dev_dbg, + dev_emerg, + dev_err, + dev_info, + dev_notice, + dev_warn, + error::{ + code::*, + Error, + Result, // + }, + init::InPlaceInit, + pr_alert, + pr_crit, + pr_debug, + pr_emerg, + pr_err, + pr_info, + pr_notice, + pr_warn, + str::CStrExt as _, + try_init, + try_pin_init, + uaccess::UserPtr, + ThisModule, // +}; + +// `super::std_vendor` is hidden, which makes the macro inline for some reason. +#[doc(no_inline)] +pub use super::dbg; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs new file mode 100644 index 000000000..0d62beeed --- /dev/null +++ b/rust/kernel/print.rs @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Printing facilities. +//! +//! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h) +//! +//! Reference: + +use crate::{ + ffi::{c_char, c_void}, + fmt, + prelude::*, + str::RawFormatter, + sync::atomic::{ + Atomic, + AtomicType, + Relaxed, // + }, +}; + +// Called from `vsprintf` with format specifier `%pA`. +#[expect(clippy::missing_safety_doc)] +#[export] +unsafe extern "C" fn rust_fmt_argument( + buf: *mut c_char, + end: *mut c_char, + ptr: *const c_void, +) -> *mut c_char { + use fmt::Write; + // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`. + let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) }; + // SAFETY: TODO. + let _ = w.write_fmt(unsafe { *ptr.cast::>() }); + w.pos().cast() +} + +/// Format strings. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +pub mod format_strings { + /// The length we copy from the `KERN_*` kernel prefixes. + const LENGTH_PREFIX: usize = 2; + + /// The length of the fixed format strings. + pub const LENGTH: usize = 10; + + /// Generates a fixed format string for the kernel's [`_printk`]. + /// + /// The format string is always the same for a given level, i.e. for a + /// given `prefix`, which are the kernel's `KERN_*` constants. + /// + /// [`_printk`]: srctree/include/linux/printk.h + const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { + // Ensure the `KERN_*` macros are what we expect. + assert!(prefix[0] == b'\x01'); + if is_cont { + assert!(prefix[1] == b'c'); + } else { + assert!(prefix[1] >= b'0' && prefix[1] <= b'7'); + } + assert!(prefix[2] == b'\x00'); + + let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont { + b"%pA\0\0\0\0\0" + } else { + b"%s: %pA\0" + }; + + [ + prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5], + suffix[6], suffix[7], + ] + } + + // Generate the format strings at compile-time. + // + // This avoids the compiler generating the contents on the fly in the stack. + // + // Furthermore, `static` instead of `const` is used to share the strings + // for all the kernel. + pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG); + pub static ALERT: [u8; LENGTH] = generate(false, bindings::KERN_ALERT); + pub static CRIT: [u8; LENGTH] = generate(false, bindings::KERN_CRIT); + pub static ERR: [u8; LENGTH] = generate(false, bindings::KERN_ERR); + pub static WARNING: [u8; LENGTH] = generate(false, bindings::KERN_WARNING); + pub static NOTICE: [u8; LENGTH] = generate(false, bindings::KERN_NOTICE); + pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO); + pub static DEBUG: [u8; LENGTH] = generate(false, bindings::KERN_DEBUG); + pub static CONT: [u8; LENGTH] = generate(true, bindings::KERN_CONT); +} + +/// Prints a message via the kernel's [`_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// # Safety +/// +/// The format string must be one of the ones in [`format_strings`], and +/// the module name must be null-terminated. +/// +/// [`_printk`]: srctree/include/linux/printk.h +#[doc(hidden)] +#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] +pub unsafe fn call_printk( + format_string: &[u8; format_strings::LENGTH], + module_name: &[u8], + args: fmt::Arguments<'_>, +) { + // `_printk` does not seem to fail in any path. + #[cfg(CONFIG_PRINTK)] + // SAFETY: TODO. + unsafe { + bindings::_printk( + format_string.as_ptr(), + module_name.as_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Prints a message via the kernel's [`_printk`] for the `CONT` level. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// [`_printk`]: srctree/include/linux/printk.h +#[doc(hidden)] +#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] +pub fn call_printk_cont(args: fmt::Arguments<'_>) { + // `_printk` does not seem to fail in any path. + // + // SAFETY: The format string is fixed. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + format_strings::CONT.as_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Performs formatting and forwards the string to [`call_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +#[cfg(not(testlib))] +#[macro_export] +#[expect(clippy::crate_in_macro_def)] +macro_rules! print_macro ( + // The non-continuation cases (most of them, e.g. `INFO`). + ($format_string:path, false, $($arg:tt)+) => ( + // To remain sound, `arg`s must be expanded outside the `unsafe` block. + // Typically one would use a `let` binding for that; however, `format_args!` + // takes borrows on the arguments, but does not extend the scope of temporaries. + // Therefore, a `match` expression is used to keep them around, since + // the scrutinee is kept until the end of the `match`. + match $crate::prelude::fmt!($($arg)+) { + // SAFETY: This hidden macro should only be called by the documented + // printing macros which ensure the format string is one of the fixed + // ones. All `__LOG_PREFIX`s are null-terminated as they are generated + // by the `module!` proc macro or fixed values defined in a kernel + // crate. + args => unsafe { + $crate::print::call_printk( + &$format_string, + crate::__LOG_PREFIX, + args, + ); + } + } + ); + + // The `CONT` case. + ($format_string:path, true, $($arg:tt)+) => ( + $crate::print::call_printk_cont( + $crate::prelude::fmt!($($arg)+), + ); + ); +); + +/// Stub for doctests +#[cfg(testlib)] +#[macro_export] +macro_rules! print_macro ( + ($format_string:path, $e:expr, $($arg:tt)+) => ( + () + ); +); + +// We could use a macro to generate these macros. However, doing so ends +// up being a bit ugly: it requires the dollar token trick to escape `$` as +// well as playing with the `doc` attribute. Furthermore, they cannot be easily +// imported in the prelude due to [1]. So, for the moment, we just write them +// manually, like in the C side; while keeping most of the logic in another +// macro, i.e. [`print_macro`]. +// +// [1]: https://github.com/rust-lang/rust/issues/52234 + +/// Prints an emergency-level message (level 0). +/// +/// Use this level if the system is unusable. +/// +/// Equivalent to the kernel's [`pr_emerg`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_emerg`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_emerg +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_emerg!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_emerg ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::EMERG, false, $($arg)*) + ) +); + +/// Prints an alert-level message (level 1). +/// +/// Use this level if action must be taken immediately. +/// +/// Equivalent to the kernel's [`pr_alert`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_alert`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_alert +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_alert!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_alert ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ALERT, false, $($arg)*) + ) +); + +/// Prints a critical-level message (level 2). +/// +/// Use this level for critical conditions. +/// +/// Equivalent to the kernel's [`pr_crit`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_crit`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_crit +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_crit!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_crit ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CRIT, false, $($arg)*) + ) +); + +/// Prints an error-level message (level 3). +/// +/// Use this level for error conditions. +/// +/// Equivalent to the kernel's [`pr_err`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_err`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_err +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_err!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_err ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ERR, false, $($arg)*) + ) +); + +/// Prints a warning-level message (level 4). +/// +/// Use this level for warning conditions. +/// +/// Equivalent to the kernel's [`pr_warn`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_warn`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_warn +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_warn!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_warn ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::WARNING, false, $($arg)*) + ) +); + +/// Prints a notice-level message (level 5). +/// +/// Use this level for normal but significant conditions. +/// +/// Equivalent to the kernel's [`pr_notice`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_notice`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_notice +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_notice!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_notice ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::NOTICE, false, $($arg)*) + ) +); + +/// Prints an info-level message (level 6). +/// +/// Use this level for informational messages. +/// +/// Equivalent to the kernel's [`pr_info`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_info`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_info +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_info!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_info ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::INFO, false, $($arg)*) + ) +); + +/// Prints a debug-level message (level 7). +/// +/// Use this level for debug messages. +/// +/// Equivalent to the kernel's [`pr_debug`] macro, except that it doesn't support dynamic debug +/// yet. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_debug`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_debug +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_debug!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_debug ( + ($($arg:tt)*) => ( + if cfg!(debug_assertions) { + $crate::print_macro!($crate::print::format_strings::DEBUG, false, $($arg)*) + } + ) +); + +/// Continues a previous log message in the same line. +/// +/// Use only when continuing a previous `pr_*!` macro (e.g. [`pr_info!`]). +/// +/// Equivalent to the kernel's [`pr_cont`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_info!`]: crate::pr_info! +/// [`pr_cont`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_cont +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::pr_cont; +/// pr_info!("hello"); +/// pr_cont!(" {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_cont ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CONT, true, $($arg)*) + ) +); + +/// A lightweight `call_once` primitive. +/// +/// This structure provides the Rust equivalent of the kernel's `DO_ONCE_LITE` macro. +/// While it would be possible to implement the feature entirely as a Rust macro, +/// the functionality that can be implemented as regular functions has been +/// extracted and implemented as the `OnceLite` struct for better code maintainability. +pub struct OnceLite(Atomic); + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +enum State { + Incomplete = 0, + Complete = 1, +} + +// SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip +// transmutable to `i32`. +unsafe impl AtomicType for State { + type Repr = i32; +} + +impl OnceLite { + /// Creates a new [`OnceLite`] in the incomplete state. + #[inline(always)] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + OnceLite(Atomic::new(State::Incomplete)) + } + + /// Calls the provided function exactly once. + /// + /// There is no other synchronization between two `call_once()`s + /// except that only one will execute `f`, in other words, callers + /// should not use a failed `call_once()` as a proof that another + /// `call_once()` has already finished and the effect is observable + /// to this thread. + pub fn call_once(&self, f: F) -> bool + where + F: FnOnce(), + { + // Avoid expensive cmpxchg if already completed. + // ORDERING: `Relaxed` is used here since no synchronization is required. + let old = self.0.load(Relaxed); + if old == State::Complete { + return false; + } + + // ORDERING: `Relaxed` is used here since no synchronization is required. + let old = self.0.xchg(State::Complete, Relaxed); + if old == State::Complete { + return false; + } + + f(); + true + } +} + +/// Run the given function exactly once. +/// +/// This is equivalent to the kernel's `DO_ONCE_LITE` macro. +/// +/// # Examples +/// +/// ``` +/// kernel::do_once_lite! { +/// kernel::pr_info!("This will be printed only once\n"); +/// }; +/// ``` +#[macro_export] +macro_rules! do_once_lite { + { $($e:tt)* } => {{ + #[link_section = ".data..once"] + static ONCE: $crate::print::OnceLite = $crate::print::OnceLite::new(); + ONCE.call_once(|| { $($e)* }); + }}; +} + +/// Prints an emergency-level message (level 0) only once. +/// +/// Equivalent to the kernel's `pr_emerg_once` macro. +#[macro_export] +macro_rules! pr_emerg_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_emerg!($($arg)*) } + ) +); + +/// Prints an alert-level message (level 1) only once. +/// +/// Equivalent to the kernel's `pr_alert_once` macro. +#[macro_export] +macro_rules! pr_alert_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_alert!($($arg)*) } + ) +); + +/// Prints a critical-level message (level 2) only once. +/// +/// Equivalent to the kernel's `pr_crit_once` macro. +#[macro_export] +macro_rules! pr_crit_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_crit!($($arg)*) } + ) +); + +/// Prints an error-level message (level 3) only once. +/// +/// Equivalent to the kernel's `pr_err_once` macro. +#[macro_export] +macro_rules! pr_err_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_err!($($arg)*) } + ) +); + +/// Prints a warning-level message (level 4) only once. +/// +/// Equivalent to the kernel's `pr_warn_once` macro. +#[macro_export] +macro_rules! pr_warn_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_warn!($($arg)*) } + ) +); + +/// Prints a notice-level message (level 5) only once. +/// +/// Equivalent to the kernel's `pr_notice_once` macro. +#[macro_export] +macro_rules! pr_notice_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_notice!($($arg)*) } + ) +); + +/// Prints an info-level message (level 6) only once. +/// +/// Equivalent to the kernel's `pr_info_once` macro. +#[macro_export] +macro_rules! pr_info_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_info!($($arg)*) } + ) +); diff --git a/rust/kernel/processor.rs b/rust/kernel/processor.rs new file mode 100644 index 000000000..85b49b361 --- /dev/null +++ b/rust/kernel/processor.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Processor related primitives. +//! +//! C header: [`include/linux/processor.h`](srctree/include/linux/processor.h) + +/// Lower CPU power consumption or yield to a hyperthreaded twin processor. +/// +/// It also happens to serve as a compiler barrier. +#[inline] +pub fn cpu_relax() { + // SAFETY: Always safe to call. + unsafe { bindings::cpu_relax() } +} diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs new file mode 100644 index 000000000..82acb531b --- /dev/null +++ b/rust/kernel/ptr.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types and functions to work with pointers and addresses. + +pub mod projection; +pub use crate::project_pointer as project; + +use core::mem::{ + align_of, + size_of, // +}; +use core::num::NonZero; + +use crate::const_assert; + +/// Type representing an alignment, which is always a power of two. +/// +/// It is used to validate that a given value is a valid alignment, and to perform masking and +/// alignment operations. +/// +/// This is a temporary substitute for the [`Alignment`] nightly type from the standard library, +/// and to be eventually replaced by it. +/// +/// [`Alignment`]: https://github.com/rust-lang/rust/issues/102070 +/// +/// # Invariants +/// +/// An alignment is always a power of two. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Alignment(NonZero); + +impl Alignment { + /// Validates that `ALIGN` is a power of two at build-time, and returns an [`Alignment`] of the + /// same value. + /// + /// A build error is triggered if `ALIGN` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// let v = Alignment::new::<16>(); + /// assert_eq!(v.as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn new() -> Self { + const_assert!( + ALIGN.is_power_of_two(), + "Provided alignment is not a power of two." + ); + + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Self(unsafe { NonZero::new_unchecked(ALIGN) }) + } + + /// Validates that `align` is a power of two at runtime, and returns an + /// [`Alignment`] of the same value. + /// + /// Returns [`None`] if `align` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new_checked(16), Some(Alignment::new::<16>())); + /// assert_eq!(Alignment::new_checked(15), None); + /// assert_eq!(Alignment::new_checked(1), Some(Alignment::new::<1>())); + /// assert_eq!(Alignment::new_checked(0), None); + /// ``` + #[inline(always)] + pub const fn new_checked(align: usize) -> Option { + if align.is_power_of_two() { + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Some(Self(unsafe { NonZero::new_unchecked(align) })) + } else { + None + } + } + + /// Returns the alignment of `T`. + /// + /// This is equivalent to [`align_of`], but with the return value provided as an [`Alignment`]. + #[inline(always)] + pub const fn of() -> Self { + // This cannot panic since alignments are always powers of two. + // + // We unfortunately cannot use `new` as it would require the `generic_const_exprs` feature. + const { Alignment::new_checked(align_of::()).unwrap() } + } + + /// Returns this alignment as a [`usize`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn as_usize(self) -> usize { + self.as_nonzero().get() + } + + /// Returns this alignment as a [`NonZero`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_nonzero().get(), 16); + /// ``` + #[inline(always)] + pub const fn as_nonzero(self) -> NonZero { + // Allow the compiler to know that the value is indeed a power of two. This can help + // optimize some operations down the line, like e.g. replacing divisions by bit shifts. + if !self.0.is_power_of_two() { + // SAFETY: Per the invariants, `self.0` is always a power of two so this block will + // never be reached. + unsafe { core::hint::unreachable_unchecked() } + } + self.0 + } + + /// Returns the base-2 logarithm of the alignment. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::of::().log2(), 0); + /// assert_eq!(Alignment::new::<16>().log2(), 4); + /// ``` + #[inline(always)] + pub const fn log2(self) -> u32 { + self.0.ilog2() + } + + /// Returns the mask for this alignment. + /// + /// This is equivalent to `!(self.as_usize() - 1)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<0x10>().mask(), !0xf); + /// ``` + #[inline(always)] + pub const fn mask(self) -> usize { + // No underflow can occur as the alignment is guaranteed to be a power of two, and thus is + // non-zero. + !(self.as_usize() - 1) + } +} + +/// Trait for items that can be aligned against an [`Alignment`]. +pub trait Alignable: Sized { + /// Aligns `self` down to `alignment`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x2f_usize.align_down(Alignment::new::<0x10>()), 0x20); + /// assert_eq!(0x30usize.align_down(Alignment::new::<0x10>()), 0x30); + /// assert_eq!(0xf0u8.align_down(Alignment::new::<0x1000>()), 0x0); + /// ``` + fn align_down(self, alignment: Alignment) -> Self; + + /// Aligns `self` up to `alignment`, returning `None` if aligning would result in an overflow. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x4fusize.align_up(Alignment::new::<0x10>()), Some(0x50)); + /// assert_eq!(0x40usize.align_up(Alignment::new::<0x10>()), Some(0x40)); + /// assert_eq!(0x0usize.align_up(Alignment::new::<0x10>()), Some(0x0)); + /// assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None); + /// assert_eq!(0x10u8.align_up(Alignment::new::<0x100>()), None); + /// assert_eq!(0x0u8.align_up(Alignment::new::<0x100>()), Some(0x0)); + /// ``` + fn align_up(self, alignment: Alignment) -> Option; +} + +/// Implement [`Alignable`] for unsigned integer types. +macro_rules! impl_alignable_uint { + ($($t:ty),*) => { + $( + impl Alignable for $t { + #[inline(always)] + fn align_down(self, alignment: Alignment) -> Self { + // The operands of `&` need to be of the same type so convert the alignment to + // `Self`. This means we need to compute the mask ourselves. + ::core::num::NonZero::::try_from(alignment.as_nonzero()) + .map(|align| self & !(align.get() - 1)) + // An alignment larger than `Self` always aligns down to `0`. + .unwrap_or(0) + } + + #[inline(always)] + fn align_up(self, alignment: Alignment) -> Option { + let aligned_down = self.align_down(alignment); + if self == aligned_down { + Some(aligned_down) + } else { + Self::try_from(alignment.as_usize()) + .ok() + .and_then(|align| aligned_down.checked_add(align)) + } + } + } + )* + }; +} + +impl_alignable_uint!(u8, u16, u32, u64, usize); + +/// Trait to represent compile-time known size information. +/// +/// This is a generalization of [`size_of`] that works for dynamically sized types. +pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. + fn size(p: *const Self) -> usize; +} + +impl KnownSize for T { + const MIN_SIZE: usize = size_of::(); + const MIN_ALIGN: Alignment = Alignment::of::(); + + #[inline(always)] + fn size(_: *const Self) -> usize { + size_of::() + } +} + +impl KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + p.len() * size_of::() + } +} + +/// Aligns `value` up to `align`. +/// +/// This is the const-compatible equivalent of [`Alignable::align_up`]. +/// +/// Returns [`None`] on overflow. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// ptr::{ +/// const_align_up, +/// Alignment, // +/// }, +/// sizes::SZ_4K, // +/// }; +/// +/// assert_eq!(const_align_up(0x4f, Alignment::new::<16>()), Some(0x50)); +/// assert_eq!(const_align_up(0x40, Alignment::new::<16>()), Some(0x40)); +/// assert_eq!(const_align_up(1, Alignment::new::()), Some(SZ_4K)); +/// ``` +#[inline(always)] +pub const fn const_align_up(value: usize, align: Alignment) -> Option { + match value.checked_add(align.as_usize() - 1) { + Some(v) => Some(v & align.mask()), + None => None, + } +} diff --git a/rust/kernel/ptr/projection.rs b/rust/kernel/ptr/projection.rs new file mode 100644 index 000000000..af72d3b0e --- /dev/null +++ b/rust/kernel/ptr/projection.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for handling projections. + +use core::{ + mem::MaybeUninit, + ops::Deref, // +}; + +use crate::prelude::*; + +/// Error raised when a projection is attempted on an array or slice out of bounds. +pub struct OutOfBound; + +impl From for Error { + #[inline(always)] + fn from(_: OutOfBound) -> Self { + ERANGE + } +} + +/// A helper trait to perform index projection. +/// +/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and +/// fallibly. +/// +/// # Safety +/// +/// For a given input pointer `slice` and return value `output`, the implementation of `index`, +/// `build_index` and `get` (if [`Some`] is returned) must ensure that: +/// - `output` has the same provenance as `slice`; +/// - `output.byte_offset_from(slice)` is between 0 to +/// `KnownSize::size(slice) - KnownSize::size(output)`. +/// +/// This means that if the input pointer is valid, then the pointer returned by `get`, `index` +/// or `build_index` is also valid. +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] +#[doc(hidden)] +pub unsafe trait ProjectIndex: Sized { + type Output: ?Sized; + + /// Returns an index-projected pointer, if in bounds. + fn get(self, slice: *mut T) -> Option<*mut Self::Output>; + + /// Returns an index-projected pointer; panic if out of bounds. + fn index(self, slice: *mut T) -> *mut Self::Output; + + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. + #[inline(always)] + fn build_index(self, slice: *mut T) -> *mut Self::Output { + match Self::get(self, slice) { + Some(v) => v, + None => build_error!(), + } + } +} + +// Forward array impl to slice impl. +// +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T; N]> for I +where + I: ProjectIndex<[T]>, +{ + type Output = >::Output; + + #[inline(always)] + fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> { + >::get(self, slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T; N]) -> *mut Self::Output { + >::index(self, slice) + } + + #[inline(always)] + fn build_index(self, slice: *mut [T; N]) -> *mut Self::Output { + >::build_index(self, slice) + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl ProjectIndex<[T]> for usize { + type Output = T; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut T> { + if self >= slice.len() { + None + } else { + Some(slice.cast::().wrapping_add(self)) + } + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut T { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + let () = zst_slice[self]; + slice.cast::().wrapping_add(self) + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl ProjectIndex<[T]> for core::ops::Range { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + let new_len = self.end.checked_sub(self.start)?; + if self.end > slice.len() { + return None; + } + Some(core::ptr::slice_from_raw_parts_mut( + slice.cast::().wrapping_add(self.start), + new_len, + )) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + _ = zst_slice[self.clone()]; + + // SAFETY: Bounds checked. + unsafe { self.get(slice).unwrap_unchecked() } + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T]> for core::ops::RangeTo { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (0..self.end).get(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (0..self.end).index(slice) + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T]> for core::ops::RangeFrom { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (self.start..slice.len()).get(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (self.start..slice.len()).index(slice) + } +} + +// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. +unsafe impl ProjectIndex<[T]> for core::ops::RangeFull { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + Some(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + slice + } +} + +/// A helper trait to perform field projection. +/// +/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that +/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being +/// used as base of projection, as they can inject unsoundness. Users therefore must not specify +/// `DEREF` and should always leave it to be inferred. +/// +/// # Safety +/// +/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`] +/// describes. +#[doc(hidden)] +pub unsafe trait ProjectField { + /// Project a pointer to a type to a pointer of a field. + /// + /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to + /// fields using `&raw mut`. + /// + /// This is needed because `base` might not point to a valid allocation, while `&raw mut` + /// requires pointers to be in bounds of a valid allocation. + /// + /// # Safety + /// + /// `f` must return a pointer in bounds of the provided pointer. + unsafe fn proj(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F; +} + +// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot +// currently support that as we need to obtain a valid allocation that `&raw const` can operate on. +// +// SAFETY: `proj` invokes `f` with valid allocation. +unsafe impl ProjectField for T { + #[inline(always)] + unsafe fn proj(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + // Create a valid allocation to start projection, as `base` is not necessarily so. The + // memory is never actually used so it will be optimized out, so it should work even for + // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also + // annotate `f` closure with `#[inline(always)]` in the macro. + let mut place = MaybeUninit::uninit(); + let place_base = place.as_mut_ptr(); + let field = f(place_base); + // SAFETY: `field` is in bounds from `base` per safety requirement. + let offset = unsafe { field.byte_offset_from(place_base) }; + // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation. + base.wrapping_byte_offset(offset).cast() + } +} + +// SAFETY: Vacuously satisfied. +unsafe impl ProjectField for T { + #[inline(always)] + unsafe fn proj(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + build_error!("this function is a guard against `Deref` impl and is never invoked"); + } +} + +/// Create a projection from a raw pointer. +/// +/// The projected pointer is within the memory region marked by the input pointer. There is no +/// requirement that the input raw pointer needs to be valid, so this macro may be used for +/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input +/// pointer is valid, the projected pointer is also valid. +/// +/// Supported projections include field projections and index projections. +/// It is not allowed to project into types that implement custom [`Deref`] or +/// [`Index`](core::ops::Index). +/// +/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an +/// expression that evaluates to a raw pointer which serves as the base of projection. `projection` +/// can be a projection expression of form `.field` (normally identifier, or numeral in case of +/// tuple structs) or of form `[index]`. +/// +/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. +/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. +/// +/// The `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. +/// The syntax is of the form `[: index]` where `flavor` indicates the way of handling +/// index out-of-bounds errors. +/// - `try` will raise an [`OutOfBound`] error (which is convertible to [`ERANGE`]). +/// - `build` will use the [`build_assert!`] mechanism to have the compiler validate the index is +/// in bounds. +/// - `panic` will cause a Rust [`panic!`] if the index goes out of bounds. +/// +/// # Examples +/// +/// Field projections are performed with `.field_name`: +/// +/// ``` +/// struct MyStruct { field: u32, } +/// let ptr: *const MyStruct = core::ptr::dangling(); +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field); +/// +/// struct MyTupleStruct(u32, u32); +/// +/// fn proj(ptr: *const MyTupleStruct) { +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1); +/// } +/// ``` +/// +/// Index projections are performed with `[: index]`, where `flavor` is `try`, `build` or +/// `panic`: +/// +/// ``` +/// fn proj(ptr: *const [u8; 32]) -> Result { +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [build: 1]); +/// // The following invocation, if uncommented, would fail the build. +/// // +/// // kernel::ptr::project!(ptr, [build: 128]); +/// +/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). +/// kernel::ptr::project!(ptr, [try: 128]); +/// +/// // This will panic at runtime if executed. +/// kernel::ptr::project!(ptr, [panic: 128]); +/// Ok(()) +/// } +/// ``` +/// +/// If you need to match on the error instead of propagate, put the invocation inside a closure: +/// +/// ``` +/// let ptr: *const [u8; 32] = core::ptr::dangling(); +/// let field_ptr: Result<*const u8> = (|| -> Result<_> { +/// Ok(kernel::ptr::project!(ptr, [try: 128])) +/// })(); +/// assert!(field_ptr.is_err()); +/// ``` +/// +/// For mutable pointers, put `mut` as the first token in macro invocation. +/// +/// ``` +/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [build: 1].1); +/// ``` +#[macro_export] +macro_rules! project_pointer { + (@gen $ptr:ident, ) => {}; + // Field projection. `$field` needs to be `tt` to support tuple index like `.0`. + (@gen $ptr:ident, .$field:tt $($rest:tt)*) => { + // SAFETY: The provided closure always returns an in-bounds pointer. + let $ptr = unsafe { + $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| { + // Check unaligned field. Not all users (e.g. DMA) can handle unaligned + // projections. + if false { + let _ = &(*ptr).$field; + } + // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the + // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`). + &raw mut (*ptr).$field + }) + }; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Fallible index projection. + (@gen $ptr:ident, [try: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) + .ok_or($crate::ptr::projection::OutOfBound)?; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Panicking index projection. + (@gen $ptr:ident, [panic: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Build-time checked index projection. + (@gen $ptr:ident, [build: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::build_index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + + (mut $ptr:expr, $($proj:tt)*) => {{ + let ptr: *mut _ = $ptr; + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr + }}; + ($ptr:expr, $($proj:tt)*) => {{ + let ptr = <*const _>::cast_mut($ptr); + // We currently always project using mutable pointer, as it is not decided whether `&raw + // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`). + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr.cast_const() + }}; +} diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs new file mode 100644 index 000000000..8aa47304b --- /dev/null +++ b/rust/kernel/pwm.rs @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2025 Samsung Electronics Co., Ltd. +// Author: Michal Wilczynski + +//! PWM subsystem abstractions. +//! +//! C header: [`include/linux/pwm.h`](srctree/include/linux/pwm.h). + +use crate::{ + bindings, + container_of, + device::{self, Bound}, + devres, + error::{self, to_result}, + prelude::*, + sync::aref::{ARef, AlwaysRefCounted}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::NonNull, // +}; + +/// Represents a PWM waveform configuration. +/// Mirrors struct [`struct pwm_waveform`](srctree/include/linux/pwm.h). +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct Waveform { + /// Total duration of one complete PWM cycle, in nanoseconds. + pub period_length_ns: u64, + + /// Duty-cycle active time, in nanoseconds. + /// + /// For a typical normal polarity configuration (active-high) this is the + /// high time of the signal. + pub duty_length_ns: u64, + + /// Duty-cycle start offset, in nanoseconds. + /// + /// Delay from the beginning of the period to the first active edge. + /// In most simple PWM setups this is `0`, so the duty cycle starts + /// immediately at each period’s start. + pub duty_offset_ns: u64, +} + +impl From for Waveform { + fn from(wf: bindings::pwm_waveform) -> Self { + Waveform { + period_length_ns: wf.period_length_ns, + duty_length_ns: wf.duty_length_ns, + duty_offset_ns: wf.duty_offset_ns, + } + } +} + +impl From for bindings::pwm_waveform { + fn from(wf: Waveform) -> Self { + bindings::pwm_waveform { + period_length_ns: wf.period_length_ns, + duty_length_ns: wf.duty_length_ns, + duty_offset_ns: wf.duty_offset_ns, + } + } +} + +/// Describes the outcome of a `round_waveform` operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoundingOutcome { + /// The requested waveform was achievable exactly or by rounding values down. + ExactOrRoundedDown, + + /// The requested waveform could only be achieved by rounding up. + RoundedUp, +} + +/// Wrapper for a PWM device [`struct pwm_device`](srctree/include/linux/pwm.h). +#[repr(transparent)] +pub struct Device(Opaque); + +impl Device { + /// Creates a reference to a [`Device`] from a valid C pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Device`] reference. + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::pwm_device) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Device` type being transparent makes the cast ok. + unsafe { &*ptr.cast::() } + } + + /// Returns a raw pointer to the underlying `pwm_device`. + fn as_raw(&self) -> *mut bindings::pwm_device { + self.0.get() + } + + /// Gets the hardware PWM index for this device within its chip. + pub fn hwpwm(&self) -> u32 { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).hwpwm } + } + + /// Gets a reference to the parent `Chip` that this device belongs to. + pub fn chip(&self) -> &Chip { + // SAFETY: `self.as_raw()` provides a valid pointer. (*self.as_raw()).chip + // is assumed to be a valid pointer to `pwm_chip` managed by the kernel. + // Chip::from_raw's safety conditions must be met. + unsafe { Chip::::from_raw((*self.as_raw()).chip) } + } + + /// Gets the label for this PWM device, if any. + pub fn label(&self) -> Option<&CStr> { + // SAFETY: self.as_raw() provides a valid pointer. + let label_ptr = unsafe { (*self.as_raw()).label }; + if label_ptr.is_null() { + return None; + } + + // SAFETY: label_ptr is non-null and points to a C string + // managed by the kernel, valid for the lifetime of the PWM device. + Some(unsafe { CStr::from_char_ptr(label_ptr) }) + } + + /// Sets the PWM waveform configuration and enables the PWM signal. + pub fn set_waveform(&self, wf: &Waveform, exact: bool) -> Result { + let c_wf = bindings::pwm_waveform::from(*wf); + + // SAFETY: `self.as_raw()` provides a valid `*mut pwm_device` pointer. + // `&c_wf` is a valid pointer to a `pwm_waveform` struct. The C function + // handles all necessary internal locking. + to_result(unsafe { bindings::pwm_set_waveform_might_sleep(self.as_raw(), &c_wf, exact) }) + } + + /// Queries the hardware for the configuration it would apply for a given + /// request. + pub fn round_waveform(&self, wf: &mut Waveform) -> Result { + let mut c_wf = bindings::pwm_waveform::from(*wf); + + // SAFETY: `self.as_raw()` provides a valid `*mut pwm_device` pointer. + // `&mut c_wf` is a valid pointer to a mutable `pwm_waveform` struct that + // the C function will update. + let ret = unsafe { bindings::pwm_round_waveform_might_sleep(self.as_raw(), &mut c_wf) }; + + to_result(ret)?; + + *wf = Waveform::from(c_wf); + + if ret == 1 { + Ok(RoundingOutcome::RoundedUp) + } else { + Ok(RoundingOutcome::ExactOrRoundedDown) + } + } + + /// Reads the current waveform configuration directly from the hardware. + pub fn get_waveform(&self) -> Result { + let mut c_wf = bindings::pwm_waveform::default(); + + // SAFETY: `self.as_raw()` is a valid pointer. We provide a valid pointer + // to a stack-allocated `pwm_waveform` struct for the kernel to fill. + to_result(unsafe { bindings::pwm_get_waveform_might_sleep(self.as_raw(), &mut c_wf) })?; + + Ok(Waveform::from(c_wf)) + } +} + +/// The result of a `round_waveform_tohw` operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RoundedWaveform { + /// A status code, 0 for success or 1 if values were rounded up. + pub status: c_int, + /// The driver-specific hardware representation of the waveform. + pub hardware_waveform: WfHw, +} + +/// Trait defining the operations for a PWM driver. +pub trait PwmOps: 'static + Send + Sync + Sized { + /// The driver-specific hardware representation of a waveform. + /// + /// This type must be [`Copy`], [`Default`], and fit within `PWM_WFHWSIZE`. + type WfHw: Copy + Default; + + /// Optional hook for when a PWM device is requested. + fn request(_chip: &Chip, _pwm: &Device, _parent_dev: &device::Device) -> Result { + Ok(()) + } + + /// Optional hook for capturing a PWM signal. + fn capture( + _chip: &Chip, + _pwm: &Device, + _result: &mut bindings::pwm_capture, + _timeout: usize, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } + + /// Convert a generic waveform to the hardware-specific representation. + /// This is typically a pure calculation and does not perform I/O. + fn round_waveform_tohw( + _chip: &Chip, + _pwm: &Device, + _wf: &Waveform, + ) -> Result> { + Err(ENOTSUPP) + } + + /// Convert a hardware-specific representation back to a generic waveform. + /// This is typically a pure calculation and does not perform I/O. + fn round_waveform_fromhw( + _chip: &Chip, + _pwm: &Device, + _wfhw: &Self::WfHw, + _wf: &mut Waveform, + ) -> Result { + Err(ENOTSUPP) + } + + /// Read the current hardware configuration into the hardware-specific representation. + fn read_waveform( + _chip: &Chip, + _pwm: &Device, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } + + /// Write a hardware-specific waveform configuration to the hardware. + fn write_waveform( + _chip: &Chip, + _pwm: &Device, + _wfhw: &Self::WfHw, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } +} + +/// Bridges Rust `PwmOps` to the C `pwm_ops` vtable. +struct Adapter { + _p: PhantomData, +} + +impl Adapter { + const VTABLE: PwmOpsVTable = create_pwm_ops::(); + + /// # Safety + /// + /// `wfhw_ptr` must be valid for writes of `size_of::()` bytes. + unsafe fn serialize_wfhw(wfhw: &T::WfHw, wfhw_ptr: *mut c_void) -> Result { + let size = core::mem::size_of::(); + + build_assert!(size <= bindings::PWM_WFHWSIZE as usize); + + // SAFETY: The caller ensures `wfhw_ptr` is valid for `size` bytes. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref::(wfhw).cast::(), + wfhw_ptr.cast::(), + size, + ) + }; + + Ok(()) + } + + /// # Safety + /// + /// `wfhw_ptr` must be valid for reads of `size_of::()` bytes. + unsafe fn deserialize_wfhw(wfhw_ptr: *const c_void) -> Result { + let size = core::mem::size_of::(); + + build_assert!(size <= bindings::PWM_WFHWSIZE as usize); + + let mut wfhw = T::WfHw::default(); + // SAFETY: The caller ensures `wfhw_ptr` is valid for `size` bytes. + unsafe { + core::ptr::copy_nonoverlapping( + wfhw_ptr.cast::(), + core::ptr::from_mut::(&mut wfhw).cast::(), + size, + ) + }; + + Ok(wfhw) + } + + /// # Safety + /// + /// `dev` must be a valid pointer to a `bindings::device` embedded within a + /// `bindings::pwm_chip`. This function is called by the device core when the + /// last reference to the device is dropped. + unsafe extern "C" fn release_callback(dev: *mut bindings::device) { + // SAFETY: The function's contract guarantees that `dev` points to a `device` + // field embedded within a valid `pwm_chip`. `container_of!` can therefore + // safely calculate the address of the containing struct. + let c_chip_ptr = unsafe { container_of!(dev, bindings::pwm_chip, dev) }; + + // SAFETY: `c_chip_ptr` is a valid pointer to a `pwm_chip` as established + // above. Calling this FFI function is safe. + let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; + + // SAFETY: The driver data was initialized in `new`. We run its destructor here. + unsafe { core::ptr::drop_in_place(drvdata_ptr.cast::()) }; + + // Now, call the original release function to free the `pwm_chip` itself. + // SAFETY: `dev` is the valid pointer passed into this callback, which is + // the expected argument for `pwmchip_release`. + unsafe { bindings::pwmchip_release(dev) }; + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn request_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + ) -> c_int { + // SAFETY: PWM core guarentees `chip_ptr` and `pwm_ptr` are valid pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::request(chip, pwm, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn capture_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + res: *mut bindings::pwm_capture, + timeout: usize, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm, result) = unsafe { + ( + Chip::::from_raw(chip_ptr), + Device::from_raw(pwm_ptr), + &mut *res, + ) + }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::capture(chip, pwm, result, timeout, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn round_waveform_tohw_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wf_ptr: *const bindings::pwm_waveform, + wfhw_ptr: *mut c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm, wf) = unsafe { + ( + Chip::::from_raw(chip_ptr), + Device::from_raw(pwm_ptr), + Waveform::from(*wf_ptr), + ) + }; + match T::round_waveform_tohw(chip, pwm, &wf) { + Ok(rounded) => { + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + if unsafe { Self::serialize_wfhw(&rounded.hardware_waveform, wfhw_ptr) }.is_err() { + return EINVAL.to_errno(); + } + rounded.status + } + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn round_waveform_fromhw_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *const c_void, + wf_ptr: *mut bindings::pwm_waveform, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + // SAFETY: `deserialize_wfhw`'s safety contract is met by this function's contract. + let wfhw = match unsafe { Self::deserialize_wfhw(wfhw_ptr) } { + Ok(v) => v, + Err(e) => return e.to_errno(), + }; + + let mut rust_wf = Waveform::default(); + match T::round_waveform_fromhw(chip, pwm, &wfhw, &mut rust_wf) { + Ok(()) => { + // SAFETY: `wf_ptr` is guaranteed valid by the C caller. + unsafe { *wf_ptr = rust_wf.into() }; + 0 + } + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn read_waveform_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *mut c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::read_waveform(chip, pwm, bound_parent) { + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + Ok(wfhw) => match unsafe { Self::serialize_wfhw(&wfhw, wfhw_ptr) } { + Ok(()) => 0, + Err(e) => e.to_errno(), + }, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn write_waveform_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *const c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + let wfhw = match unsafe { Self::deserialize_wfhw(wfhw_ptr) } { + Ok(v) => v, + Err(e) => return e.to_errno(), + }; + match T::write_waveform(chip, pwm, &wfhw, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } +} + +/// VTable structure wrapper for PWM operations. +/// Mirrors [`struct pwm_ops`](srctree/include/linux/pwm.h). +#[repr(transparent)] +pub struct PwmOpsVTable(bindings::pwm_ops); + +// SAFETY: PwmOpsVTable is Send. The vtable contains only function pointers +// and a size, which are simple data types that can be safely moved across +// threads. The thread-safety of calling these functions is handled by the +// kernel's locking mechanisms. +unsafe impl Send for PwmOpsVTable {} + +// SAFETY: PwmOpsVTable is Sync. The vtable is immutable after it is created, +// so it can be safely referenced and accessed concurrently by multiple threads +// e.g. to read the function pointers. +unsafe impl Sync for PwmOpsVTable {} + +impl PwmOpsVTable { + /// Returns a raw pointer to the underlying `pwm_ops` struct. + pub(crate) fn as_raw(&self) -> *const bindings::pwm_ops { + &self.0 + } +} + +/// Creates a PWM operations vtable for a type `T` that implements `PwmOps`. +/// +/// This is used to bridge Rust trait implementations to the C `struct pwm_ops` +/// expected by the kernel. +pub const fn create_pwm_ops() -> PwmOpsVTable { + let mut ops: bindings::pwm_ops = pin_init::zeroed(); + + ops.request = Some(Adapter::::request_callback); + ops.capture = Some(Adapter::::capture_callback); + + ops.round_waveform_tohw = Some(Adapter::::round_waveform_tohw_callback); + ops.round_waveform_fromhw = Some(Adapter::::round_waveform_fromhw_callback); + ops.read_waveform = Some(Adapter::::read_waveform_callback); + ops.write_waveform = Some(Adapter::::write_waveform_callback); + ops.sizeof_wfhw = core::mem::size_of::(); + + PwmOpsVTable(ops) +} + +/// Wrapper for a PWM chip/controller ([`struct pwm_chip`](srctree/include/linux/pwm.h)). +#[repr(transparent)] +pub struct Chip(Opaque, PhantomData); + +impl Chip { + /// Creates a reference to a [`Chip`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Chip`] reference. + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::pwm_chip) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Chip` type being transparent makes the cast ok. + unsafe { &*ptr.cast::() } + } + + /// Returns a raw pointer to the underlying `pwm_chip`. + pub(crate) fn as_raw(&self) -> *mut bindings::pwm_chip { + self.0.get() + } + + /// Gets the number of PWM channels (hardware PWMs) on this chip. + pub fn num_channels(&self) -> u32 { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).npwm } + } + + /// Returns `true` if the chip supports atomic operations for configuration. + pub fn is_atomic(&self) -> bool { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).atomic } + } + + /// Returns a reference to the embedded `struct device` abstraction. + pub fn device(&self) -> &device::Device { + // SAFETY: + // - `self.as_raw()` provides a valid pointer to `bindings::pwm_chip`. + // - The `dev` field is an instance of `bindings::device` embedded + // within `pwm_chip`. + // - Taking a pointer to this embedded field is valid. + // - `device::Device` is `#[repr(transparent)]`. + // - The lifetime of the returned reference is tied to `self`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } + + /// Gets the typed driver specific data associated with this chip's embedded device. + pub fn drvdata(&self) -> &T { + // SAFETY: `pwmchip_get_drvdata` returns the pointer to the private data area, + // which we know holds our `T`. The pointer is valid for the lifetime of `self`. + unsafe { &*bindings::pwmchip_get_drvdata(self.as_raw()).cast::() } + } + + /// Returns a reference to the parent device of this PWM chip's device. + /// + /// # Safety + /// + /// The caller must guarantee that the parent device exists and is bound. + /// This is guaranteed by the PWM core during `PwmOps` callbacks. + unsafe fn bound_parent_device(&self) -> &device::Device { + // SAFETY: Per the function's safety contract, the parent device exists. + let parent = unsafe { self.device().parent().unwrap_unchecked() }; + + // SAFETY: Per the function's safety contract, the parent device is bound. + // This is guaranteed by the PWM core during `PwmOps` callbacks. + unsafe { parent.as_bound() } + } + + /// Allocates and wraps a PWM chip using `bindings::pwmchip_alloc`. + /// + /// Returns an [`ARef`] managing the chip's lifetime via refcounting + /// on its embedded `struct device`. + #[allow(clippy::new_ret_no_self)] + pub fn new<'a>( + parent_dev: &'a device::Device, + num_channels: u32, + data: impl pin_init::PinInit, + ) -> Result> { + let sizeof_priv = core::mem::size_of::(); + // SAFETY: `pwmchip_alloc` allocates memory for the C struct and our private data. + let c_chip_ptr_raw = + unsafe { bindings::pwmchip_alloc(parent_dev.as_raw(), num_channels, sizeof_priv) }; + + let c_chip_ptr: *mut bindings::pwm_chip = error::from_err_ptr(c_chip_ptr_raw)?; + + // SAFETY: The `drvdata` pointer is the start of the private area, which is where + // we will construct our `T` object. + let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; + + // SAFETY: We construct the `T` object in-place in the allocated private memory. + unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| { + // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained + // from `pwmchip_alloc()`. We will not use pointer after this. + unsafe { bindings::pwmchip_put(c_chip_ptr) } + })?; + + // SAFETY: `c_chip_ptr` points to a valid chip. + unsafe { (*c_chip_ptr).dev.release = Some(Adapter::::release_callback) }; + + // SAFETY: `c_chip_ptr` points to a valid chip. + // The `Adapter`'s `VTABLE` has a 'static lifetime, so the pointer + // returned by `as_raw()` is always valid. + unsafe { (*c_chip_ptr).ops = Adapter::::VTABLE.as_raw() }; + + // Cast the `*mut bindings::pwm_chip` to `*mut Chip`. This is valid because + // `Chip` is `repr(transparent)` over `Opaque`, and + // `Opaque` is `repr(transparent)` over `T`. + let chip_ptr_as_self = c_chip_ptr.cast::(); + + // SAFETY: `chip_ptr_as_self` points to a valid `Chip` (layout-compatible with + // `bindings::pwm_chip`) whose embedded device has refcount 1. + // `ARef::from_raw` takes this pointer and manages it via `AlwaysRefCounted`. + let chip = unsafe { ARef::from_raw(NonNull::new_unchecked(chip_ptr_as_self)) }; + + Ok(UnregisteredChip { chip, parent_dev }) + } +} + +// SAFETY: Implements refcounting for `Chip` using the embedded `struct device`. +unsafe impl AlwaysRefCounted for Chip { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self.0.get()` points to a valid `pwm_chip` because `self` exists. + // The embedded `dev` is valid. `get_device` increments its refcount. + unsafe { bindings::get_device(&raw mut (*self.0.get()).dev) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull>) { + let c_chip_ptr = obj.cast::().as_ptr(); + + // SAFETY: `obj` is a valid pointer to a `Chip` (and thus `bindings::pwm_chip`) + // with a non-zero refcount. `put_device` handles decrement and final release. + unsafe { bindings::put_device(&raw mut (*c_chip_ptr).dev) }; + } +} + +// SAFETY: `Chip` is a wrapper around `*mut bindings::pwm_chip`. The underlying C +// structure's state is managed and synchronized by the kernel's device model +// and PWM core locking mechanisms. Therefore, it is safe to move the `Chip` +// wrapper (and the pointer it contains) across threads. +unsafe impl Send for Chip {} + +// SAFETY: It is safe for multiple threads to have shared access (`&Chip`) because +// the `Chip` data is immutable from the Rust side without holding the appropriate +// kernel locks, which the C core is responsible for. Any interior mutability is +// handled and synchronized by the C kernel code. +unsafe impl Sync for Chip {} + +/// A wrapper around `ARef>` that ensures that `register` can only be called once. +pub struct UnregisteredChip<'a, T: PwmOps> { + chip: ARef>, + parent_dev: &'a device::Device, +} + +impl UnregisteredChip<'_, T> { + /// Registers a PWM chip with the PWM subsystem. + /// + /// Transfers its ownership to the `devres` framework, which ties its lifetime + /// to the parent device. + /// On unbind of the parent device, the `devres` entry will be dropped, automatically + /// calling `pwmchip_remove`. This function should be called from the driver's `probe`. + pub fn register(self) -> Result>> { + let c_chip_ptr = self.chip.as_raw(); + + // SAFETY: `c_chip_ptr` points to a valid chip with its ops initialized. + // `__pwmchip_add` is the C function to register the chip with the PWM core. + to_result(unsafe { bindings::__pwmchip_add(c_chip_ptr, core::ptr::null_mut()) })?; + + let registration = Registration { + chip: ARef::clone(&self.chip), + }; + + devres::register(self.parent_dev, registration, GFP_KERNEL)?; + + Ok(self.chip) + } +} + +impl Deref for UnregisteredChip<'_, T> { + type Target = Chip; + + fn deref(&self) -> &Self::Target { + &self.chip + } +} + +/// A resource guard that ensures `pwmchip_remove` is called on drop. +/// +/// This struct is intended to be managed by the `devres` framework by transferring its ownership +/// via [`devres::register`]. This ties the lifetime of the PWM chip registration +/// to the lifetime of the underlying device. +struct Registration { + chip: ARef>, +} + +impl Drop for Registration { + fn drop(&mut self) { + let chip_raw = self.chip.as_raw(); + + // SAFETY: `chip_raw` points to a chip that was successfully registered. + // `bindings::pwmchip_remove` is the correct C function to unregister it. + // This `drop` implementation is called automatically by `devres` on driver unbind. + unsafe { bindings::pwmchip_remove(chip_raw) }; + } +} + +/// Declares a kernel module that exposes a single PWM driver. +/// +/// # Examples +/// +///```ignore +/// kernel::module_pwm_platform_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +///``` +#[macro_export] +macro_rules! module_pwm_platform_driver { + ($($user_args:tt)*) => { + $crate::module_platform_driver! { + $($user_args)* + imports_ns: ["PWM"], + } + }; +} diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs new file mode 100644 index 000000000..6fbd579d4 --- /dev/null +++ b/rust/kernel/rbtree.rs @@ -0,0 +1,1433 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Red-black trees. +//! +//! C header: [`include/linux/rbtree.h`](srctree/include/linux/rbtree.h) +//! +//! Reference: + +use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; +use core::{ + cmp::{Ord, Ordering}, + marker::PhantomData, + mem::MaybeUninit, + ptr::{addr_of_mut, from_mut, NonNull}, +}; + +/// A red-black tree with owned nodes. +/// +/// It is backed by the kernel C red-black trees. +/// +/// # Examples +/// +/// In the example below we do several operations on a tree. We note that insertions may fail if +/// the system is out of memory. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// assert_eq!(tree.get(&10), Some(&100)); +/// assert_eq!(tree.get(&20), Some(&200)); +/// assert_eq!(tree.get(&30), Some(&300)); +/// } +/// +/// // Iterate over the nodes we just inserted. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Print all elements. +/// for (key, value) in &tree { +/// pr_info!("{} = {}\n", key, value); +/// } +/// +/// // Replace one of the elements. +/// tree.try_create_and_insert(10, 1000, flags::GFP_KERNEL)?; +/// +/// // Check that the tree reflects the replacement. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Change the value of one of the elements. +/// *tree.get_mut(&30).unwrap() = 3000; +/// +/// // Check that the tree reflects the update. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Remove an element. +/// tree.remove(&10); +/// +/// // Check that the tree reflects the removal. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); +/// assert!(iter.next().is_none()); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// In the example below, we first allocate a node, acquire a spinlock, then insert the node into +/// the tree. This is useful when the insertion context does not allow sleeping, for example, when +/// holding a spinlock. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode}, sync::SpinLock}; +/// +/// fn insert_test(tree: &SpinLock>) -> Result { +/// // Pre-allocate node. This may fail (as it allocates memory). +/// let node = RBTreeNode::new(10, 100, flags::GFP_KERNEL)?; +/// +/// // Insert node while holding the lock. It is guaranteed to succeed with no allocation +/// // attempts. +/// let mut guard = tree.lock(); +/// guard.insert(node); +/// Ok(()) +/// } +/// ``` +/// +/// In the example below, we reuse an existing node allocation from an element we removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Remove a node, getting back ownership of it. +/// let existing = tree.remove(&30); +/// +/// // Check that the tree reflects the removal. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Create a preallocated reservation that we can re-use later. +/// let reservation = RBTreeNodeReservation::new(flags::GFP_KERNEL)?; +/// +/// // Insert a new node into the tree, reusing the previous allocation. This is guaranteed to +/// // succeed (no memory allocations). +/// tree.insert(reservation.into_node(15, 150)); +/// +/// // Check that the tree reflect the new insertion. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&15, &150))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert!(iter.next().is_none()); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// Non-null parent/children pointers stored in instances of the `rb_node` C struct are always +/// valid, and pointing to a field of our internal representation of a node. +pub struct RBTree { + root: bindings::rb_root, + _p: PhantomData>, +} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Send condition as would be used for a struct with K and V fields. +unsafe impl Send for RBTree {} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Sync condition as would be used for a struct with K and V fields. +unsafe impl Sync for RBTree {} + +impl RBTree { + /// Creates a new and empty tree. + pub fn new() -> Self { + Self { + // INVARIANT: There are no nodes in the tree, so the invariant holds vacuously. + root: bindings::rb_root::default(), + _p: PhantomData, + } + } + + /// Returns true if this tree is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.root.rb_node.is_null() + } + + /// Returns an iterator over the tree nodes, sorted by key. + pub fn iter(&self) -> Iter<'_, K, V> { + Iter { + _tree: PhantomData, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(&self.root) }, + _phantom: PhantomData, + }, + } + } + + /// Returns a mutable iterator over the tree nodes, sorted by key. + pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { + IterMut { + _tree: PhantomData, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(from_mut(&mut self.root)) }, + _phantom: PhantomData, + }, + } + } + + /// Returns an iterator over the keys of the nodes in the tree, in sorted order. + pub fn keys(&self) -> impl Iterator { + self.iter().map(|(k, _)| k) + } + + /// Returns an iterator over the values of the nodes in the tree, sorted by key. + pub fn values(&self) -> impl Iterator { + self.iter().map(|(_, v)| v) + } + + /// Returns a mutable iterator over the values of the nodes in the tree, sorted by key. + pub fn values_mut(&mut self) -> impl Iterator { + self.iter_mut().map(|(_, v)| v) + } + + /// Returns a cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front_mut(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns an immutable cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back_mut(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } +} + +impl RBTree +where + K: Ord, +{ + /// Tries to insert a new value into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// Returns an error if it cannot allocate memory for the new node. + pub fn try_create_and_insert( + &mut self, + key: K, + value: V, + flags: Flags, + ) -> Result>> { + Ok(self.insert(RBTreeNode::new(key, value, flags)?)) + } + + /// Inserts a new node into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// This function always succeeds. + pub fn insert(&mut self, node: RBTreeNode) -> Option> { + match self.raw_entry(&node.node.key) { + RawEntry::Occupied(entry) => Some(entry.replace(node)), + RawEntry::Vacant(entry) => { + entry.insert(node); + None + } + } + } + + fn raw_entry(&mut self, key: &K) -> RawEntry<'_, K, V> { + let raw_self: *mut RBTree = self; + // The returned `RawEntry` is used to call either `rb_link_node` or `rb_replace_node`. + // The parameters of `bindings::rb_link_node` are as follows: + // - `node`: A pointer to an uninitialized node being inserted. + // - `parent`: A pointer to an existing node in the tree. One of its child pointers must be + // null, and `node` will become a child of `parent` by replacing that child pointer + // with a pointer to `node`. + // - `rb_link`: A pointer to either the left-child or right-child field of `parent`. This + // specifies which child of `parent` should hold `node` after this call. The + // value of `*rb_link` must be null before the call to `rb_link_node`. If the + // red/black tree is empty, then it’s also possible for `parent` to be null. In + // this case, `rb_link` is a pointer to the `root` field of the red/black tree. + // + // We will traverse the tree looking for a node that has a null pointer as its child, + // representing an empty subtree where we can insert our new node. We need to make sure + // that we preserve the ordering of the nodes in the tree. In each iteration of the loop + // we store `parent` and `child_field_of_parent`, and the new `node` will go somewhere + // in the subtree of `parent` that `child_field_of_parent` points at. Once + // we find an empty subtree, we can insert the new node using `rb_link_node`. + let mut parent = core::ptr::null_mut(); + let mut child_field_of_parent: &mut *mut bindings::rb_node = + // SAFETY: `raw_self` is a valid pointer to the `RBTree` (created from `self` above). + unsafe { &mut (*raw_self).root.rb_node }; + while !(*child_field_of_parent).is_null() { + let curr = *child_field_of_parent; + // SAFETY: All links fields we create are in a `Node`. + let node = unsafe { container_of!(curr, Node, links) }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + match key.cmp(unsafe { &(*node).key }) { + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Less => child_field_of_parent = unsafe { &mut (*curr).rb_left }, + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Greater => child_field_of_parent = unsafe { &mut (*curr).rb_right }, + Ordering::Equal => { + return RawEntry::Occupied(OccupiedEntry { + rbtree: self, + node_links: curr, + }) + } + } + parent = curr; + } + + RawEntry::Vacant(RawVacantEntry { + rbtree: raw_self, + parent, + child_field_of_parent, + _phantom: PhantomData, + }) + } + + /// Gets the given key's corresponding entry in the map for in-place manipulation. + pub fn entry(&mut self, key: K) -> Entry<'_, K, V> { + match self.raw_entry(&key) { + RawEntry::Occupied(entry) => Entry::Occupied(entry), + RawEntry::Vacant(entry) => Entry::Vacant(VacantEntry { raw: entry, key }), + } + } + + /// Used for accessing the given node, if it exists. + pub fn find_mut(&mut self, key: &K) -> Option> { + match self.raw_entry(key) { + RawEntry::Occupied(entry) => Some(entry), + RawEntry::Vacant(_entry) => None, + } + } + + /// Returns a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option<&V> { + let mut node = self.root.rb_node; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }; + + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + let this_ref = unsafe { &*this }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let node_ref = unsafe { &*node }; + + node = match key.cmp(&this_ref.key) { + Ordering::Less => node_ref.rb_left, + Ordering::Greater => node_ref.rb_right, + Ordering::Equal => return Some(&this_ref.value), + } + } + None + } + + /// Returns a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + self.find_mut(key).map(|node| node.into_mut()) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the node that was removed if one exists, or [`None`] otherwise. + pub fn remove_node(&mut self, key: &K) -> Option> { + self.find_mut(key).map(OccupiedEntry::remove_node) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the value that was removed if one exists, or [`None`] otherwise. + pub fn remove(&mut self, key: &K) -> Option { + self.find_mut(key).map(OccupiedEntry::remove) + } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound_mut(&mut self, key: &K) -> Option> + where + K: Ord, + { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound(&self, key: &K) -> Option> + where + K: Ord, + { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } + + fn find_best_match(&self, key: &K) -> Option> { + let mut node = self.root.rb_node; + let mut best_key: Option<&K> = None; + let mut best_links: Option> = None; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }; + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + let this_key = unsafe { &(*this).key }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let node_ref = unsafe { &*node }; + + match key.cmp(this_key) { + Ordering::Equal => { + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); + break; + } + Ordering::Greater => { + node = node_ref.rb_right; + } + Ordering::Less => { + let is_better_match = match best_key { + None => true, + Some(best) => best > this_key, + }; + if is_better_match { + best_key = Some(this_key); + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); + } + node = node_ref.rb_left; + } + }; + } + best_links + } +} + +impl Default for RBTree { + fn default() -> Self { + Self::new() + } +} + +impl Drop for RBTree { + fn drop(&mut self) { + // SAFETY: `root` is valid as it's embedded in `self` and we have a valid `self`. + let mut next = unsafe { bindings::rb_first_postorder(&self.root) }; + + // INVARIANT: The loop invariant is that all tree nodes from `next` in postorder are valid. + while !next.is_null() { + // SAFETY: All links fields we create are in a `Node`. + let this = unsafe { container_of!(next, Node, links) }; + + // Find out what the next node is before disposing of the current one. + // SAFETY: `next` and all nodes in postorder are still valid. + next = unsafe { bindings::rb_next_postorder(next) }; + + // INVARIANT: This is the destructor, so we break the type invariant during clean-up, + // but it is not observable. The loop invariant is still maintained. + + // SAFETY: `this` is valid per the loop invariant. + unsafe { drop(KBox::from_raw(this)) }; + } + } +} + +/// A bidirectional mutable cursor over the tree nodes, sorted by key. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Move the cursor, updating it to the 2nd element. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Peek at the next element without impacting the cursor. +/// let next = cursor.peek_next().unwrap(); +/// assert_eq!(next, (&30, &300)); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Moving past the last element causes the cursor to return [`None`]. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// let cursor = cursor.move_next(); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// A cursor can also be obtained at the last element in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// let mut cursor = tree.cursor_back_mut().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Obtaining a cursor returns [`None`] if the tree is empty. +/// +/// ``` +/// use kernel::rbtree::RBTree; +/// +/// let mut tree: RBTree = RBTree::new(); +/// assert!(tree.cursor_front_mut().is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`RBTree::cursor_lower_bound`] can be used to start at an arbitrary node in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert five elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(40, 400, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(50, 500, flags::GFP_KERNEL)?; +/// +/// // If the provided key exists, a cursor to that key is returned. +/// let cursor = tree.cursor_lower_bound(&20).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // If the provided key doesn't exist, a cursor to the first larger element in sort order is returned. +/// let cursor = tree.cursor_lower_bound(&25).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // If there is no larger key, [`None`] is returned. +/// let cursor = tree.cursor_lower_bound(&55); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The cursor allows mutation of values in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Retrieve a cursor. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// +/// // Get a mutable reference to the current value. +/// let (k, v) = cursor.current_mut(); +/// *v = 1000; +/// +/// // The updated value is reflected in the tree. +/// let updated = tree.get(&10).unwrap(); +/// assert_eq!(updated, &1000); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// It also allows node removal. The following examples demonstrate the behavior of removing the current node. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Remove the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// cursor = cursor.remove_current().0.unwrap(); +/// +/// // If a node exists after the current element, it is returned. +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Get a cursor to the last element, and remove it. +/// cursor = tree.cursor_back_mut().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Since there is no next node, the previous node is returned. +/// cursor = cursor.remove_current().0.unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Removing the last element in the tree returns [`None`]. +/// assert!(cursor.remove_current().0.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Nodes adjacent to the current node can also be removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_prev` from the first element returns [`None`]. +/// assert!(cursor.remove_prev().is_none()); +/// +/// // Get a cursor to the last element. +/// cursor = tree.cursor_back_mut().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Calling `remove_prev` removes and returns the middle element. +/// assert_eq!(cursor.remove_prev().unwrap().to_key_value(), (20, 200)); +/// +/// // Calling `remove_next` from the last element returns [`None`]. +/// assert!(cursor.remove_next().is_none()); +/// +/// // Move to the first element +/// cursor = cursor.move_prev().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_next` removes and returns the last element. +/// assert_eq!(cursor.remove_next().unwrap().to_key_value(), (30, 300)); +/// +/// # Ok::<(), Error>(()) +/// +/// ``` +/// +/// # Invariants +/// - `current` points to a node that is in the same [`RBTree`] as `tree`. +pub struct CursorMut<'a, K, V> { + tree: &'a mut RBTree, + current: NonNull, +} + +/// A bidirectional immutable cursor over the tree nodes, sorted by key. This is a simpler +/// variant of [`CursorMut`] that is basically providing read only access. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let cursor = tree.cursor_front().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct Cursor<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + current: NonNull, +} + +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. +unsafe impl<'a, K: Sync, V: Sync> Send for Cursor<'a, K, V> {} + +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. +unsafe impl<'a, K: Sync, V: Sync> Sync for Cursor<'a, K, V> {} + +impl<'a, K, V> Cursor<'a, K, V> { + /// The current node + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of `'b`. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { &(*this).value }; + (k, v) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the next node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } +} + +// SAFETY: The [`CursorMut`] has exclusive access to both `K` and `V`, so it is sufficient to +// require them to be `Send`. +// The cursor only gives out immutable references to the keys, but since it has exclusive access to +// those same keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the +// user. +unsafe impl<'a, K: Send, V: Send> Send for CursorMut<'a, K, V> {} + +// SAFETY: The [`CursorMut`] gives out immutable references to `K` and mutable references to `V`, +// so it has the same thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for CursorMut<'a, K, V> {} + +impl<'a, K, V> CursorMut<'a, K, V> { + /// The current node. + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + + /// The current node, with a mutable value + pub fn current_mut(&mut self) -> (&K, &mut V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an mutable reference by the function signature. + unsafe { Self::to_key_value_mut(self.current) } + } + + /// Remove the current node from the tree. + /// + /// Returns a tuple where the first element is a cursor to the next node, if it exists, + /// else the previous node, else [`None`] (if the tree becomes empty). The second element + /// is the removed node. + pub fn remove_current(self) -> (Option, RBTreeNode) { + let prev = self.get_neighbor_raw(Direction::Prev); + let next = self.get_neighbor_raw(Direction::Next); + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(self.current.as_ptr(), Node, links) }; + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { KBox::from_raw(this) }; + let node = RBTreeNode { node }; + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(&mut (*this).links, addr_of_mut!(self.tree.root)) }; + + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self.tree`. + let cursor = next.or(prev).map(|current| Self { + current, + tree: self.tree, + }); + + (cursor, node) + } + + /// Remove the previous node, returning it if it exists. + pub fn remove_prev(&mut self) -> Option> { + self.remove_neighbor(Direction::Prev) + } + + /// Remove the next node, returning it if it exists. + pub fn remove_next(&mut self) -> Option> { + self.remove_neighbor(Direction::Next) + } + + fn remove_neighbor(&mut self, direction: Direction) -> Option> { + if let Some(neighbor) = self.get_neighbor_raw(direction) { + let neighbor = neighbor.as_ptr(); + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(neighbor, addr_of_mut!(self.tree.root)) }; + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(neighbor, Node, links) }; + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { KBox::from_raw(this) }; + return Some(RBTreeNode { node }); + } + None + } + + /// Move the cursor to the previous node, returning [`None`] if it doesn't exist. + pub fn move_prev(self) -> Option { + self.mv(Direction::Prev) + } + + /// Move the cursor to the next node, returning [`None`] if it doesn't exist. + pub fn move_next(self) -> Option { + self.mv(Direction::Next) + } + + fn mv(self, direction: Direction) -> Option { + // INVARIANT: + // - `neighbor` is a valid node in the [`RBTree`] pointed to by `self.tree`. + self.get_neighbor_raw(direction).map(|neighbor| Self { + tree: self.tree, + current: neighbor, + }) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the next node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + /// Access the previous node mutably without moving the cursor. + pub fn peek_prev_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Prev) + } + + /// Access the next node mutably without moving the cursor. + pub fn peek_next_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Next) + } + + fn peek_mut(&mut self, direction: Direction) -> Option<(&K, &mut V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have a mutable reference to `self`. + unsafe { Self::to_key_value_mut(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of `'b`. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees immutable access to `node`. + (k, unsafe { &*v }) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has mutable access to `node` for the duration of `'b`. + unsafe fn to_key_value_mut<'b>(node: NonNull) -> (&'b K, &'b mut V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees mutable access to `node`. + (k, unsafe { &mut *v }) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to the key for the duration of `'b`. + unsafe fn to_key_value_raw<'b>(node: NonNull) -> (&'b K, *mut V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { addr_of_mut!((*this).value) }; + (k, v) + } +} + +/// Direction for [`Cursor`] and [`CursorMut`] operations. +enum Direction { + /// the node immediately before, in sort order + Prev, + /// the node immediately after, in sort order + Next, +} + +impl<'a, K, V> IntoIterator for &'a RBTree { + type Item = (&'a K, &'a V); + type IntoIter = Iter<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter`]. +pub struct Iter<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + iter_raw: IterRaw, +} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Send for Iter<'a, K, V> {} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for Iter<'a, K, V> {} + +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + // SAFETY: Due to `self._tree`, `k` and `v` are valid for the lifetime of `'a`. + self.iter_raw.next().map(|(k, v)| unsafe { (&*k, &*v) }) + } +} + +impl<'a, K, V> IntoIterator for &'a mut RBTree { + type Item = (&'a K, &'a mut V); + type IntoIter = IterMut<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// A mutable iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter_mut`]. +pub struct IterMut<'a, K, V> { + _tree: PhantomData<&'a mut RBTree>, + iter_raw: IterRaw, +} + +// SAFETY: The [`IterMut`] has exclusive access to both `K` and `V`, so it is sufficient to require them to be `Send`. +// The iterator only gives out immutable references to the keys, but since the iterator has exclusive access to those same +// keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the user. +unsafe impl<'a, K: Send, V: Send> Send for IterMut<'a, K, V> {} + +// SAFETY: The [`IterMut`] gives out immutable references to K and mutable references to V, so it has the same +// thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for IterMut<'a, K, V> {} + +impl<'a, K, V> Iterator for IterMut<'a, K, V> { + type Item = (&'a K, &'a mut V); + + fn next(&mut self) -> Option { + self.iter_raw.next().map(|(k, v)| + // SAFETY: Due to `&mut self`, we have exclusive access to `k` and `v`, for the lifetime of `'a`. + unsafe { (&*k, &mut *v) }) + } +} + +/// A raw iterator over the nodes of a [`RBTree`]. +/// +/// # Invariants +/// - `self.next` is a valid pointer. +/// - `self.next` points to a node stored inside of a valid `RBTree`. +struct IterRaw { + next: *mut bindings::rb_node, + _phantom: PhantomData (K, V)>, +} + +impl Iterator for IterRaw { + type Item = (*mut K, *mut V); + + fn next(&mut self) -> Option { + if self.next.is_null() { + return None; + } + + // SAFETY: By the type invariant of `IterRaw`, `self.next` is a valid node in an `RBTree`, + // and by the type invariant of `RBTree`, all nodes point to the links field of `Node` objects. + let cur = unsafe { container_of!(self.next, Node, links) }; + + // SAFETY: `self.next` is a valid tree node by the type invariants. + self.next = unsafe { bindings::rb_next(self.next) }; + + // SAFETY: By the same reasoning above, it is safe to dereference the node. + Some(unsafe { (addr_of_mut!((*cur).key), addr_of_mut!((*cur).value)) }) + } +} + +/// A memory reservation for a red-black tree node. +/// +/// +/// It contains the memory needed to hold a node that can be inserted into a red-black tree. One +/// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). +pub struct RBTreeNodeReservation { + node: KBox>>, +} + +impl RBTreeNodeReservation { + /// Allocates memory for a node to be eventually initialised and inserted into the tree via a + /// call to [`RBTree::insert`]. + pub fn new(flags: Flags) -> Result> { + Ok(RBTreeNodeReservation { + node: KBox::new_uninit(flags)?, + }) + } +} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. Those can always +// be moved across threads. +unsafe impl Send for RBTreeNodeReservation {} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. +unsafe impl Sync for RBTreeNodeReservation {} + +impl RBTreeNodeReservation { + /// Initialises a node reservation. + /// + /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. + pub fn into_node(self, key: K, value: V) -> RBTreeNode { + let node = KBox::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); + RBTreeNode { node } + } +} + +/// A red-black tree node. +/// +/// The node is fully initialised (with key and value) and can be inserted into a tree without any +/// extra allocations or failure paths. +pub struct RBTreeNode { + node: KBox>, +} + +impl RBTreeNode { + /// Allocates and initialises a node that can be inserted into the tree via + /// [`RBTree::insert`]. + pub fn new(key: K, value: V, flags: Flags) -> Result> { + Ok(RBTreeNodeReservation::new(flags)?.into_node(key, value)) + } + + /// Get the key and value from inside the node. + pub fn to_key_value(self) -> (K, V) { + let node = KBox::into_inner(self.node); + + (node.key, node.value) + } +} + +// SAFETY: If K and V can be sent across threads, then it's also okay to send [`RBTreeNode`] across +// threads. +unsafe impl Send for RBTreeNode {} + +// SAFETY: If K and V can be accessed without synchronization, then it's also okay to access +// [`RBTreeNode`] without synchronization. +unsafe impl Sync for RBTreeNode {} + +impl RBTreeNode { + /// Drop the key and value, but keep the allocation. + /// + /// It then becomes a reservation that can be re-initialised into a different node (i.e., with + /// a different key and/or value). + /// + /// The existing key and value are dropped in-place as part of this operation, that is, memory + /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). + pub fn into_reservation(self) -> RBTreeNodeReservation { + RBTreeNodeReservation { + node: KBox::drop_contents(self.node), + } + } +} + +/// A view into a single entry in a map, which may either be vacant or occupied. +/// +/// This enum is constructed from the [`RBTree::entry`]. +/// +/// [`entry`]: fn@RBTree::entry +pub enum Entry<'a, K, V> { + /// This [`RBTree`] does not have a node with this key. + Vacant(VacantEntry<'a, K, V>), + /// This [`RBTree`] already has a node with this key. + Occupied(OccupiedEntry<'a, K, V>), +} + +/// Like [`Entry`], except that it doesn't have ownership of the key. +enum RawEntry<'a, K, V> { + Vacant(RawVacantEntry<'a, K, V>), + Occupied(OccupiedEntry<'a, K, V>), +} + +/// A view into a vacant entry in a [`RBTree`]. It is part of the [`Entry`] enum. +pub struct VacantEntry<'a, K, V> { + key: K, + raw: RawVacantEntry<'a, K, V>, +} + +/// Like [`VacantEntry`], but doesn't hold on to the key. +/// +/// # Invariants +/// - `parent` may be null if the new node becomes the root. +/// - `child_field_of_parent` is a valid pointer to the left-child or right-child of `parent`. If `parent` is +/// null, it is a pointer to the root of the [`RBTree`]. +struct RawVacantEntry<'a, K, V> { + rbtree: *mut RBTree, + /// The node that will become the parent of the new node if we insert one. + parent: *mut bindings::rb_node, + /// This points to the left-child or right-child field of `parent`, or `root` if `parent` is + /// null. + child_field_of_parent: *mut *mut bindings::rb_node, + _phantom: PhantomData<&'a mut RBTree>, +} + +impl<'a, K, V> RawVacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + /// + /// The `node` must have a key such that inserting it here does not break the ordering of this + /// [`RBTree`]. + fn insert(self, node: RBTreeNode) -> &'a mut V { + let node = KBox::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when + // the node is removed or replaced. + let node_links = unsafe { addr_of_mut!((*node).links) }; + + // INVARIANT: We are linking in a new node, which is valid. It remains valid because we + // "forgot" it with `KBox::into_raw`. + // SAFETY: The type invariants of `RawVacantEntry` are exactly the safety requirements of `rb_link_node`. + unsafe { bindings::rb_link_node(node_links, self.parent, self.child_field_of_parent) }; + + // SAFETY: All pointers are valid. `node` has just been inserted into the tree. + unsafe { bindings::rb_insert_color(node_links, addr_of_mut!((*self.rbtree).root)) }; + + // SAFETY: The node is valid until we remove it from the tree. + unsafe { &mut (*node).value } + } +} + +impl<'a, K, V> VacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + pub fn insert(self, value: V, reservation: RBTreeNodeReservation) -> &'a mut V { + self.raw.insert(reservation.into_node(self.key, value)) + } +} + +/// A view into an occupied entry in a [`RBTree`]. It is part of the [`Entry`] enum. +/// +/// # Invariants +/// - `node_links` is a valid, non-null pointer to a tree node in `self.rbtree` +pub struct OccupiedEntry<'a, K, V> { + rbtree: &'a mut RBTree, + /// The node that this entry corresponds to. + node_links: *mut bindings::rb_node, +} + +impl<'a, K, V> OccupiedEntry<'a, K, V> { + /// Gets a reference to the value in the entry. + pub fn get(&self) -> &V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have shared access to the underlying tree, and can thus give out a shared reference. + unsafe { &(*container_of!(self.node_links, Node, links)).value } + } + + /// Gets a mutable reference to the value in the entry. + pub fn get_mut(&mut self) -> &mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have exclusive access to the underlying tree, and can thus give out a mutable reference. + unsafe { &mut (*(container_of!(self.node_links, Node, links))).value } + } + + /// Converts the entry into a mutable reference to its value. + /// + /// If you need multiple references to the `OccupiedEntry`, see [`self#get_mut`]. + pub fn into_mut(self) -> &'a mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - This consumes the `&'a mut RBTree`, therefore it can give out a mutable reference that lives for `'a`. + unsafe { &mut (*(container_of!(self.node_links, Node, links))).value } + } + + /// Remove this entry from the [`RBTree`]. + pub fn remove_node(self) -> RBTreeNode { + // SAFETY: The node is a node in the tree, so it is valid. + unsafe { bindings::rb_erase(self.node_links, &mut self.rbtree.root) }; + + // INVARIANT: The node is being returned and the caller may free it, however, it was + // removed from the tree. So the invariants still hold. + RBTreeNode { + // SAFETY: The node was a node in the tree, but we removed it, so we can convert it + // back into a box. + node: unsafe { KBox::from_raw(container_of!(self.node_links, Node, links)) }, + } + } + + /// Takes the value of the entry out of the map, and returns it. + pub fn remove(self) -> V { + let rb_node = self.remove_node(); + let node = KBox::into_inner(rb_node.node); + + node.value + } + + /// Swap the current node for the provided node. + /// + /// The key of both nodes must be equal. + fn replace(self, node: RBTreeNode) -> RBTreeNode { + let node = KBox::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when + // the node is removed or replaced. + let new_node_links = unsafe { addr_of_mut!((*node).links) }; + + // SAFETY: This updates the pointers so that `new_node_links` is in the tree where + // `self.node_links` used to be. + unsafe { + bindings::rb_replace_node(self.node_links, new_node_links, &mut self.rbtree.root) + }; + + // SAFETY: + // - `self.node_ptr` produces a valid pointer to a node in the tree. + // - Now that we removed this entry from the tree, we can convert the node to a box. + let old_node = unsafe { KBox::from_raw(container_of!(self.node_links, Node, links)) }; + + RBTreeNode { node: old_node } + } +} + +struct Node { + links: bindings::rb_node, + key: K, + value: V, +} diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 000000000..41e730ced --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Regulator abstractions, providing a standard kernel interface to control +//! voltage and current regulators. +//! +//! The intention is to allow systems to dynamically control regulator power +//! output in order to save power and prolong battery life. This applies to both +//! voltage regulators (where voltage output is controllable) and current sinks +//! (where current limit is controllable). +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Regulators are modeled in Rust with a collection of states. Each state may +//! enforce a given invariant, and they may convert between each other where applicable. +//! +//! See [Voltage and current regulator API](https://docs.kernel.org/driver-api/regulator.html) +//! for more information. + +use crate::{ + bindings, + device::{Bound, Device}, + error::{from_err_ptr, to_result, Result}, + prelude::*, +}; + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, // +}; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Enabled {} + impl Sealed for super::Disabled {} +} + +/// A trait representing the different states a [`Regulator`] can be in. +pub trait RegulatorState: private::Sealed + 'static { + /// Whether the regulator should be disabled when dropped. + const DISABLE_ON_DROP: bool; +} + +/// A state where the [`Regulator`] is known to be enabled. +/// +/// The `enable` reference count held by this state is decremented when it is +/// dropped. +pub struct Enabled; + +/// A state where this [`Regulator`] handle has not specifically asked for the +/// underlying regulator to be enabled. This means that this reference does not +/// own an `enable` reference count, but the regulator may still be on. +pub struct Disabled; + +impl RegulatorState for Enabled { + const DISABLE_ON_DROP: bool = true; +} + +impl RegulatorState for Disabled { + const DISABLE_ON_DROP: bool = false; +} + +/// A trait that abstracts the ability to check if a [`Regulator`] is enabled. +pub trait IsEnabled: RegulatorState {} +impl IsEnabled for Disabled {} + +/// An error that can occur when trying to convert a [`Regulator`] between states. +pub struct Error { + /// The error that occurred. + pub error: kernel::error::Error, + + /// The regulator that caused the error, so that the operation may be retried. + pub regulator: Regulator, +} +/// Obtains and enables a [`devres`]-managed regulator for a device. +/// +/// This calls [`regulator_disable()`] and [`regulator_put()`] automatically on +/// driver detach. +/// +/// This API is identical to `devm_regulator_get_enable()`, and should be +/// preferred over the [`Regulator`] API if the caller only +/// cares about the regulator being enabled. +/// +/// [`devres`]: https://docs.kernel.org/driver-api/driver-model/devres.html +/// [`regulator_disable()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_disable +/// [`regulator_put()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_put +pub fn devm_enable(dev: &Device, name: &CStr) -> Result { + // SAFETY: `dev` is a valid and bound device, while `name` is a valid C + // string. + to_result(unsafe { bindings::devm_regulator_get_enable(dev.as_raw(), name.as_char_ptr()) }) +} + +/// Same as [`devm_enable`], but calls `devm_regulator_get_enable_optional` +/// instead. +/// +/// This obtains and enables a [`devres`]-managed regulator for a device, but +/// does not print a message nor provides a dummy if the regulator is not found. +/// +/// This calls [`regulator_disable()`] and [`regulator_put()`] automatically on +/// driver detach. +/// +/// [`devres`]: https://docs.kernel.org/driver-api/driver-model/devres.html +/// [`regulator_disable()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_disable +/// [`regulator_put()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_put +pub fn devm_enable_optional(dev: &Device, name: &CStr) -> Result { + // SAFETY: `dev` is a valid and bound device, while `name` is a valid C + // string. + to_result(unsafe { + bindings::devm_regulator_get_enable_optional(dev.as_raw(), name.as_char_ptr()) + }) +} + +/// A `struct regulator` abstraction. +/// +/// # Examples +/// +/// ## Enabling a regulator +/// +/// This example uses [`Regulator`], which is suitable for drivers that +/// enable a regulator at probe time and leave them on until the device is +/// removed or otherwise shutdown. +/// +/// These users can store [`Regulator`] directly in their driver's +/// private data struct. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Disabled, Enabled}; +/// fn enable(dev: &Device, min_voltage: Voltage, max_voltage: Voltage) -> Result { +/// // Obtain a reference to a (fictitious) regulator. +/// let regulator: Regulator = Regulator::::get(dev, c"vcc")?; +/// +/// // The voltage can be set before enabling the regulator if needed, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Enables the regulator, consuming the previous value. +/// // +/// // From now on, the regulator is known to be enabled because of the type +/// // `Enabled`. +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator = +/// regulator.try_into_enabled().map_err(|error| error.error)?; +/// +/// // The voltage can also be set after enabling the regulator, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// A more concise shortcut is available for enabling a regulator. This is +/// equivalent to `regulator_get_enable()`: +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Enabled}; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. +/// let regulator: Regulator = Regulator::::get(dev, c"vcc")?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// If a driver only cares about the regulator being on for as long it is bound +/// to a device, then it should use [`devm_enable`] or [`devm_enable_optional`]. +/// This should be the default use-case unless more fine-grained control over +/// the regulator's state is required. +/// +/// [`devm_enable`]: crate::regulator::devm_enable +/// [`devm_optional`]: crate::regulator::devm_enable_optional +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::{Bound, Device}; +/// # use kernel::regulator; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. This +/// // call only returns whether the operation succeeded. +/// regulator::devm_enable(dev, c"vcc")?; +/// +/// // The regulator will be disabled and put when `dev` is unbound. +/// Ok(()) +/// } +/// ``` +/// +/// ## Disabling a regulator +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Enabled, Disabled}; +/// fn disable(dev: &Device, regulator: Regulator) -> Result { +/// // We can also disable an enabled regulator without reliquinshing our +/// // refcount: +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator = +/// regulator.try_into_disabled().map_err(|error| error.error)?; +/// +/// // The refcount will be decremented when `regulator` is dropped. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// # Invariants +/// +/// - `inner` is a pointer obtained from a successful call to +/// [`regulator_get()`]. It is treated as an opaque token that may only be +/// accessed using C API methods (e.g., it may be `NULL` if the C API returns +/// `NULL`). +/// +/// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get +pub struct Regulator +where + State: RegulatorState, +{ + inner: *mut bindings::regulator, + _phantom: PhantomData, +} + +impl Regulator { + /// Sets the voltage for the regulator. + /// + /// This can be used to ensure that the device powers up cleanly. + pub fn set_voltage(&self, min_voltage: Voltage, max_voltage: Voltage) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { + bindings::regulator_set_voltage( + self.inner, + min_voltage.as_microvolts(), + max_voltage.as_microvolts(), + ) + }) + } + + /// Gets the current voltage of the regulator. + pub fn get_voltage(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + let voltage = unsafe { bindings::regulator_get_voltage(self.inner) }; + + to_result(voltage).map(|()| Voltage::from_microvolts(voltage)) + } + + fn get_internal(dev: &Device, name: &CStr) -> Result> { + let inner = + // SAFETY: It is safe to call `regulator_get()`, on a device pointer + // received from the C code. + from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_char_ptr()) })?; + + // INVARIANT: `inner` is a pointer obtained from `regulator_get()`, and + // the call was successful. + Ok(Self { + inner, + _phantom: PhantomData, + }) + } + + fn enable_internal(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_enable(self.inner) }) + } + + fn disable_internal(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_disable(self.inner) }) + } +} + +impl Regulator { + /// Obtains a [`Regulator`] instance from the system. + pub fn get(dev: &Device, name: &CStr) -> Result { + Regulator::get_internal(dev, name) + } + + /// Attempts to convert the regulator to an enabled state. + pub fn try_into_enabled(self) -> Result, Error> { + // We will be transferring the ownership of our `regulator_get()` count to + // `Regulator`. + let regulator = ManuallyDrop::new(self); + + regulator + .enable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator { + /// Obtains a [`Regulator`] instance from the system and enables it. + /// + /// This is equivalent to calling `regulator_get_enable()` in the C API. + pub fn get(dev: &Device, name: &CStr) -> Result { + Regulator::::get_internal(dev, name)? + .try_into_enabled() + .map_err(|error| error.error) + } + + /// Attempts to convert the regulator to a disabled state. + pub fn try_into_disabled(self) -> Result, Error> { + // We will be transferring the ownership of our `regulator_get()` count + // to `Regulator`. + let regulator = ManuallyDrop::new(self); + + regulator + .disable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator { + /// Checks if the regulator is enabled. + pub fn is_enabled(&self) -> bool { + // SAFETY: Safe as per the type invariants of `Regulator`. + unsafe { bindings::regulator_is_enabled(self.inner) != 0 } + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + if T::DISABLE_ON_DROP { + // SAFETY: By the type invariants, we know that `self` owns a + // reference on the enabled refcount, so it is safe to relinquish it + // now. + unsafe { bindings::regulator_disable(self.inner) }; + } + // SAFETY: By the type invariants, we know that `self` owns a reference, + // so it is safe to relinquish it now. + unsafe { bindings::regulator_put(self.inner) }; + } +} + +// SAFETY: It is safe to send a `Regulator` across threads. In particular, a +// Regulator can be dropped from any thread. +unsafe impl Send for Regulator {} + +// SAFETY: It is safe to send a &Regulator across threads because the C side +// handles its own locking. +unsafe impl Sync for Regulator {} + +/// A voltage. +/// +/// This type represents a voltage value in microvolts. +#[repr(transparent)] +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct Voltage(i32); + +impl Voltage { + /// Creates a new `Voltage` from a value in microvolts. + pub fn from_microvolts(uv: i32) -> Self { + Self(uv) + } + + /// Returns the value of the voltage in microvolts as an [`i32`]. + pub fn as_microvolts(self) -> i32 { + self.0 + } +} diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs new file mode 100644 index 000000000..0e55e2a0f --- /dev/null +++ b/rust/kernel/revocable.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Revocable objects. +//! +//! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence +//! of a [`RevocableGuard`] ensures that objects remain valid. + +use pin_init::Wrapper; + +use crate::{ + prelude::*, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::drop_in_place, // +}; + +/// An object that can become inaccessible at runtime. +/// +/// Once access is revoked and all concurrent users complete (i.e., all existing instances of +/// [`RevocableGuard`] are dropped), the wrapped object is also dropped. +/// +/// # Examples +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = v.try_access()?; +/// Some(guard.a + guard.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +/// +/// Sample example as above, but explicitly using the rcu read side lock. +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// use kernel::sync::rcu; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = rcu::read_lock(); +/// let e = v.try_access_with_guard(&guard)?; +/// Some(e.a + e.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +#[pin_data(PinnedDrop)] +pub struct Revocable { + is_available: AtomicFlag, + #[pin] + data: Opaque, +} + +// SAFETY: `Revocable` is `Send` if the wrapped object is also `Send`. This is because while the +// functionality exposed by `Revocable` can be accessed from any thread/CPU, it is possible that +// this isn't supported by the wrapped object. +unsafe impl Send for Revocable {} + +// SAFETY: `Revocable` is `Sync` if the wrapped object is both `Send` and `Sync`. We require `Send` +// from the wrapped object as well because of `Revocable::revoke`, which can trigger the `Drop` +// implementation of the wrapped object from an arbitrary thread. +unsafe impl Sync for Revocable {} + +impl Revocable { + /// Creates a new revocable instance of the given data. + pub fn new(data: impl PinInit) -> impl PinInit { + try_pin_init!(Self { + is_available: AtomicFlag::new(true), + data <- Opaque::pin_init(data), + }? E) + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a guard that gives access to the object otherwise; the object is guaranteed to + /// remain accessible while the guard is alive. In such cases, callers are not allowed to sleep + /// because another CPU may be waiting to complete the revocation of this object. + pub fn try_access(&self) -> Option> { + let guard = rcu::read_lock(); + if self.is_available.load(Relaxed) { + // Since `self.is_available` is true, data is initialised and has to remain valid + // because the RCU read side lock prevents it from being dropped. + Some(RevocableGuard::new(self.data.get(), guard)) + } else { + None + } + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a shared reference to the object otherwise; the object is guaranteed to + /// remain accessible while the rcu read side guard is alive. In such cases, callers are not + /// allowed to sleep because another CPU may be waiting to complete the revocation of this + /// object. + pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { + if self.is_available.load(Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { &*self.data.get() }) + } else { + None + } + } + + /// Tries to access the wrapped object and run a closure on it while the guard is held. + /// + /// This is a convenience method to run short non-sleepable code blocks while ensuring the + /// guard is dropped afterwards. [`Self::try_access`] carries the risk that the caller will + /// forget to explicitly drop that returned guard before calling sleepable code; this method + /// adds an extra safety to make sure it doesn't happen. + /// + /// Returns [`None`] if the object has been revoked and is therefore no longer accessible, or + /// the result of the closure wrapped in [`Some`]. If the closure returns a [`Result`] then the + /// return type becomes `Option>`, which can be inconvenient. Users are encouraged to + /// define their own macro that turns the [`Option`] into a proper error code and flattens the + /// inner result into it if it makes sense within their subsystem. + pub fn try_access_with R>(&self, f: F) -> Option { + self.try_access().map(|t| f(&*t)) + } + + /// Directly access the revocable wrapped object. + /// + /// # Safety + /// + /// The caller must ensure this [`Revocable`] instance hasn't been revoked and won't be revoked + /// as long as the returned `&T` lives. + pub unsafe fn access(&self) -> &T { + // SAFETY: By the safety requirement of this function it is guaranteed that + // `self.data.get()` is a valid pointer to an instance of `T`. + unsafe { &*self.data.get() } + } + + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + unsafe fn revoke_internal(&self) -> bool { + let revoke = self.is_available.xchg(false, Relaxed); + + if revoke { + if SYNC { + rcu::synchronize_rcu(); + } + + // SAFETY: We know `self.data` is valid because only one CPU can succeed the + // `compare_exchange` above that takes `is_available` from `true` to `false`. + unsafe { drop_in_place(self.data.get()) }; + } + + revoke + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`], + /// expecting that there are no concurrent users of the object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + /// + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + pub unsafe fn revoke_nosync(&self) -> bool { + // SAFETY: By the safety requirement of this function, the caller ensures that nobody is + // accessing the data anymore and hence we don't have to wait for the grace period to + // finish. + unsafe { self.revoke_internal::() } + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`]. + /// + /// If there are concurrent users of the object (i.e., ones that called + /// [`Revocable::try_access`] beforehand and still haven't dropped the returned guard), this + /// function waits for the concurrent access to complete before dropping the wrapped object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + pub fn revoke(&self) -> bool { + // SAFETY: By passing `true` we ask `revoke_internal` to wait for the grace period to + // finish. + unsafe { self.revoke_internal::() } + } +} + +#[pinned_drop] +impl PinnedDrop for Revocable { + fn drop(self: Pin<&mut Self>) { + // Drop only if the data hasn't been revoked yet (in which case it has already been + // dropped). + // SAFETY: We are not moving out of `p`, only dropping in place + let p = unsafe { self.get_unchecked_mut() }; + if *p.is_available.get_mut() { + // SAFETY: We know `self.data` is valid because no other CPU has changed + // `is_available` to `false` yet, and no other CPU can do it anymore because this CPU + // holds the only reference (mutable) to `self` now. + unsafe { drop_in_place(p.data.get()) }; + } + } +} + +/// A guard that allows access to a revocable object and keeps it alive. +/// +/// CPUs may not sleep while holding on to [`RevocableGuard`] because it's in atomic context +/// holding the RCU read-side lock. +/// +/// # Invariants +/// +/// The RCU read-side lock is held while the guard is alive. +pub struct RevocableGuard<'a, T> { + // This can't use the `&'a T` type because references that appear in function arguments must + // not become dangling during the execution of the function, which can happen if the + // `RevocableGuard` is passed as a function argument and then dropped during execution of the + // function. + data_ref: *const T, + _rcu_guard: rcu::Guard, + _p: PhantomData<&'a ()>, +} + +impl RevocableGuard<'_, T> { + fn new(data_ref: *const T, rcu_guard: rcu::Guard) -> Self { + Self { + data_ref, + _rcu_guard: rcu_guard, + _p: PhantomData, + } + } +} + +impl Deref for RevocableGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariants, we hold the rcu read-side lock, so the object is + // guaranteed to remain valid. + unsafe { &*self.data_ref } + } +} diff --git a/rust/kernel/safety.rs b/rust/kernel/safety.rs new file mode 100644 index 000000000..c1c6bd0fa --- /dev/null +++ b/rust/kernel/safety.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Safety related APIs. + +/// Checks that a precondition of an unsafe function is followed. +/// +/// The check is enabled at runtime if debug assertions (`CONFIG_RUST_DEBUG_ASSERTIONS`) +/// are enabled. Otherwise, this macro is a no-op. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::unsafe_precondition_assert; +/// +/// struct RawBuffer { +/// data: [T; N], +/// } +/// +/// impl RawBuffer { +/// /// # Safety +/// /// +/// /// The caller must ensure that `index` is less than `N`. +/// unsafe fn set_unchecked(&mut self, index: usize, value: T) { +/// unsafe_precondition_assert!( +/// index < N, +/// "RawBuffer::set_unchecked() requires index ({index}) < N ({N})" +/// ); +/// +/// // SAFETY: By the safety requirements of this function, `index` is valid. +/// unsafe { +/// *self.data.get_unchecked_mut(index) = value; +/// } +/// } +/// } +/// ``` +/// +/// # Panics +/// +/// Panics if the expression is evaluated to [`false`] at runtime. +#[macro_export] +macro_rules! unsafe_precondition_assert { + ($cond:expr $(,)?) => { + $crate::unsafe_precondition_assert!(@inner $cond, ::core::stringify!($cond)) + }; + + ($cond:expr, $($arg:tt)+) => { + $crate::unsafe_precondition_assert!(@inner $cond, $crate::prelude::fmt!($($arg)+)) + }; + + (@inner $cond:expr, $msg:expr) => { + ::core::debug_assert!($cond, "unsafe precondition violated: {}", $msg) + }; +} diff --git a/rust/kernel/scatterlist.rs b/rust/kernel/scatterlist.rs new file mode 100644 index 000000000..b83c468b5 --- /dev/null +++ b/rust/kernel/scatterlist.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for scatter-gather lists. +//! +//! C header: [`include/linux/scatterlist.h`](srctree/include/linux/scatterlist.h) +//! +//! Scatter-gather (SG) I/O is a memory access technique that allows devices to perform DMA +//! operations on data buffers that are not physically contiguous in memory. It works by creating a +//! "scatter-gather list", an array where each entry specifies the address and length of a +//! physically contiguous memory segment. +//! +//! The device's DMA controller can then read this list and process the segments sequentially as +//! part of one logical I/O request. This avoids the need for a single, large, physically contiguous +//! memory buffer, which can be difficult or impossible to allocate. +//! +//! This module provides safe Rust abstractions over the kernel's `struct scatterlist` and +//! `struct sg_table` types. +//! +//! The main entry point is the [`SGTable`] type, which represents a complete scatter-gather table. +//! It can be either: +//! +//! - An owned table ([`SGTable>`]), created from a Rust memory buffer (e.g., [`VVec`]). +//! This type manages the allocation of the `struct sg_table`, the DMA mapping of the buffer, and +//! the automatic cleanup of all resources. +//! - A borrowed reference (&[`SGTable`]), which provides safe, read-only access to a table that was +//! allocated by other (e.g., C) code. +//! +//! Individual entries in the table are represented by [`SGEntry`], which can be accessed by +//! iterating over an [`SGTable`]. + +use crate::{ + alloc, + alloc::allocator::VmallocPageIter, + bindings, + device::{Bound, Device}, + devres::Devres, + dma, error, + io::ResourceSize, + page, + prelude::*, + sync::aref::ARef, + types::Opaque, +}; +use core::{ops::Deref, ptr::NonNull}; + +/// A single entry in a scatter-gather list. +/// +/// An `SGEntry` represents a single, physically contiguous segment of memory that has been mapped +/// for DMA. +/// +/// Instances of this struct are obtained by iterating over an [`SGTable`]. Drivers do not create +/// or own [`SGEntry`] objects directly. +#[repr(transparent)] +pub struct SGEntry(Opaque); + +// SAFETY: `SGEntry` can be sent to any task. +unsafe impl Send for SGEntry {} + +// SAFETY: `SGEntry` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for SGEntry {} + +impl SGEntry { + /// Convert a raw `struct scatterlist *` to a `&'a SGEntry`. + /// + /// # Safety + /// + /// Callers must ensure that the `struct scatterlist` pointed to by `ptr` is valid for the + /// lifetime `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::scatterlist) -> &'a Self { + // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer + // to a `struct scatterlist` for the duration of `'a`. + unsafe { &*ptr.cast() } + } + + /// Obtain the raw `struct scatterlist *`. + #[inline] + fn as_raw(&self) -> *mut bindings::scatterlist { + self.0.get() + } + + /// Returns the DMA address of this SG entry. + /// + /// This is the address that the device should use to access the memory segment. + #[inline] + pub fn dma_address(&self) -> dma::DmaAddress { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. + unsafe { bindings::sg_dma_address(self.as_raw()) } + } + + /// Returns the length of this SG entry in bytes. + #[inline] + pub fn dma_len(&self) -> ResourceSize { + #[allow(clippy::useless_conversion)] + // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. + unsafe { bindings::sg_dma_len(self.as_raw()) }.into() + } +} + +/// The borrowed generic type of an [`SGTable`], representing a borrowed or externally managed +/// table. +#[repr(transparent)] +pub struct Borrowed(Opaque); + +// SAFETY: `Borrowed` can be sent to any task. +unsafe impl Send for Borrowed {} + +// SAFETY: `Borrowed` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for Borrowed {} + +/// A scatter-gather table. +/// +/// This struct is a wrapper around the kernel's `struct sg_table`. It manages a list of DMA-mapped +/// memory segments that can be passed to a device for I/O operations. +/// +/// The generic parameter `T` is used as a generic type to distinguish between owned and borrowed +/// tables. +/// +/// - [`SGTable`]: An owned table created and managed entirely by Rust code. It handles +/// allocation, DMA mapping, and cleanup of all associated resources. See [`SGTable::new`]. +/// - [`SGTable`} (or simply [`SGTable`]): Represents a table whose lifetime is managed +/// externally. It can be used safely via a borrowed reference `&'a SGTable`, where `'a` is the +/// external lifetime. +/// +/// All [`SGTable`] variants can be iterated over the individual [`SGEntry`]s. +#[repr(transparent)] +#[pin_data] +pub struct SGTable { + #[pin] + inner: T, +} + +impl SGTable { + /// Creates a borrowed `&'a SGTable` from a raw `struct sg_table` pointer. + /// + /// This allows safe access to an `sg_table` that is managed elsewhere (for example, in C code). + /// + /// # Safety + /// + /// Callers must ensure that: + /// + /// - the `struct sg_table` pointed to by `ptr` is valid for the entire lifetime of `'a`, + /// - the data behind `ptr` is not modified concurrently for the duration of `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::sg_table) -> &'a Self { + // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer + // to a `struct sg_table` for the duration of `'a`. + unsafe { &*ptr.cast() } + } + + #[inline] + fn as_raw(&self) -> *mut bindings::sg_table { + self.inner.0.get() + } + + /// Returns an [`SGTableIter`] bound to the lifetime of `self`. + pub fn iter(&self) -> SGTableIter<'_> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. + let nents = unsafe { (*self.as_raw()).nents }; + + let pos = if nents > 0 { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. + let ptr = unsafe { (*self.as_raw()).sgl }; + + // SAFETY: `ptr` is guaranteed to be a valid pointer to a `struct scatterlist`. + Some(unsafe { SGEntry::from_raw(ptr) }) + } else { + None + }; + + SGTableIter { pos, nents } + } +} + +/// Represents the DMA mapping state of a `struct sg_table`. +/// +/// This is used as an inner type of [`Owned`] to manage the DMA mapping lifecycle. +/// +/// # Invariants +/// +/// - `sgt` is a valid pointer to a `struct sg_table` for the entire lifetime of the +/// [`DmaMappedSgt`]. +/// - `sgt` is always DMA mapped. +struct DmaMappedSgt { + sgt: NonNull, + dev: ARef, + dir: dma::DataDirection, +} + +// SAFETY: `DmaMappedSgt` can be sent to any task. +unsafe impl Send for DmaMappedSgt {} + +// SAFETY: `DmaMappedSgt` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for DmaMappedSgt {} + +impl DmaMappedSgt { + /// # Safety + /// + /// - `sgt` must be a valid pointer to a `struct sg_table` for the entire lifetime of the + /// returned [`DmaMappedSgt`]. + /// - The caller must guarantee that `sgt` remains DMA mapped for the entire lifetime of + /// [`DmaMappedSgt`]. + unsafe fn new( + sgt: NonNull, + dev: &Device, + dir: dma::DataDirection, + ) -> Result { + // SAFETY: + // - `dev.as_raw()` is a valid pointer to a `struct device`, which is guaranteed to be + // bound to a driver for the duration of this call. + // - `sgt` is a valid pointer to a `struct sg_table`. + error::to_result(unsafe { + bindings::dma_map_sgtable(dev.as_raw(), sgt.as_ptr(), dir.into(), 0) + })?; + + // INVARIANT: By the safety requirements of this function it is guaranteed that `sgt` is + // valid for the entire lifetime of this object instance. + Ok(Self { + sgt, + dev: dev.into(), + dir, + }) + } +} + +impl Drop for DmaMappedSgt { + #[inline] + fn drop(&mut self) { + // SAFETY: + // - `self.dev.as_raw()` is a pointer to a valid `struct device`. + // - `self.dev` is the same device the mapping has been created for in `Self::new()`. + // - `self.sgt.as_ptr()` is a valid pointer to a `struct sg_table` by the type invariants + // of `Self`. + // - `self.dir` is the same `dma::DataDirection` the mapping has been created with in + // `Self::new()`. + unsafe { + bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.as_ptr(), self.dir.into(), 0) + }; + } +} + +/// A transparent wrapper around a `struct sg_table`. +/// +/// While we could also create the `struct sg_table` in the constructor of [`Owned`], we can't tear +/// down the `struct sg_table` in [`Owned::drop`]; the drop order in [`Owned`] matters. +#[repr(transparent)] +struct RawSGTable(Opaque); + +// SAFETY: `RawSGTable` can be sent to any task. +unsafe impl Send for RawSGTable {} + +// SAFETY: `RawSGTable` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for RawSGTable {} + +impl RawSGTable { + /// # Safety + /// + /// - `pages` must be a slice of valid `struct page *`. + /// - The pages pointed to by `pages` must remain valid for the entire lifetime of the returned + /// [`RawSGTable`]. + unsafe fn new( + pages: &mut [*mut bindings::page], + size: usize, + max_segment: u32, + flags: alloc::Flags, + ) -> Result { + // `sg_alloc_table_from_pages_segment()` expects at least one page, otherwise it + // produces a NPE. + if pages.is_empty() { + return Err(EINVAL); + } + + let sgt = Opaque::zeroed(); + // SAFETY: + // - `sgt.get()` is a valid pointer to uninitialized memory. + // - As by the check above, `pages` is not empty. + error::to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + pages.as_mut_ptr(), + pages.len().try_into()?, + 0, + size, + max_segment, + flags.as_raw(), + ) + })?; + + Ok(Self(sgt)) + } + + #[inline] + fn as_raw(&self) -> *mut bindings::sg_table { + self.0.get() + } +} + +impl Drop for RawSGTable { + #[inline] + fn drop(&mut self) { + // SAFETY: `sgt` is a valid and initialized `struct sg_table`. + unsafe { bindings::sg_free_table(self.0.get()) }; + } +} + +/// The [`Owned`] generic type of an [`SGTable`]. +/// +/// A [`SGTable`] signifies that the [`SGTable`] owns all associated resources: +/// +/// - The backing memory pages. +/// - The `struct sg_table` allocation (`sgt`). +/// - The DMA mapping, managed through a [`Devres`]-managed `DmaMappedSgt`. +/// +/// Users interact with this type through the [`SGTable`] handle and do not need to manage +/// [`Owned`] directly. +#[pin_data] +pub struct Owned

{ + // Note: The drop order is relevant; we first have to unmap the `struct sg_table`, then free the + // `struct sg_table` and finally free the backing pages. + #[pin] + dma: Devres, + sgt: RawSGTable, + _pages: P, +} + +// SAFETY: `Owned` can be sent to any task if `P` can be send to any task. +unsafe impl Send for Owned

{} + +// SAFETY: `Owned` has no interior mutability and can be accessed concurrently if `P` can be +// accessed concurrently. +unsafe impl Sync for Owned

{} + +impl

Owned

+where + for<'a> P: page::AsPageIter = VmallocPageIter<'a>> + 'static, +{ + fn new( + dev: &Device, + mut pages: P, + dir: dma::DataDirection, + flags: alloc::Flags, + ) -> Result + '_> { + let page_iter = pages.page_iter(); + let size = page_iter.size(); + + let mut page_vec: KVec<*mut bindings::page> = + KVec::with_capacity(page_iter.page_count(), flags)?; + + for page in page_iter { + page_vec.push(page.as_ptr(), flags)?; + } + + // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes + // an `unsigned int`. + // + // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`. + let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { + 0 => u32::MAX, + max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX), + }; + + Ok(try_pin_init!(&this in Self { + // SAFETY: + // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`. + // - The pages contained in `pages` remain valid for the entire lifetime of the + // `RawSGTable`. + sgt: unsafe { RawSGTable::new(&mut page_vec, size, max_segment, flags) }?, + dma <- { + // SAFETY: `this` is a valid pointer to uninitialized memory. + let sgt = unsafe { &raw mut (*this.as_ptr()).sgt }.cast(); + + // SAFETY: `sgt` is guaranteed to be non-null. + let sgt = unsafe { NonNull::new_unchecked(sgt) }; + + // SAFETY: + // - It is guaranteed that the object returned by `DmaMappedSgt::new` won't out-live + // `sgt`. + // - `sgt` is never DMA unmapped manually. + Devres::new(dev, unsafe { DmaMappedSgt::new(sgt, dev, dir) }) + }, + _pages: pages, + })) + } +} + +impl

SGTable> +where + for<'a> P: page::AsPageIter = VmallocPageIter<'a>> + 'static, +{ + /// Allocates a new scatter-gather table from the given pages and maps it for DMA. + /// + /// This constructor creates a new [`SGTable`] that takes ownership of `P`. + /// It allocates a `struct sg_table`, populates it with entries corresponding to the physical + /// pages of `P`, and maps the table for DMA with the specified [`Device`] and + /// [`dma::DataDirection`]. + /// + /// The DMA mapping is managed through [`Devres`], ensuring that the DMA mapping is unmapped + /// once the associated [`Device`] is unbound, or when the [`SGTable`] is dropped. + /// + /// # Parameters + /// + /// * `dev`: The [`Device`] that will be performing the DMA. + /// * `pages`: The entity providing the backing pages. It must implement [`page::AsPageIter`]. + /// The ownership of this entity is moved into the new [`SGTable`]. + /// * `dir`: The [`dma::DataDirection`] of the DMA transfer. + /// * `flags`: Allocation flags for internal allocations (e.g., [`GFP_KERNEL`]). + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// device::{Bound, Device}, + /// dma, page, + /// prelude::*, + /// scatterlist::{SGTable, Owned}, + /// }; + /// + /// fn test(dev: &Device) -> Result { + /// let size = 4 * page::PAGE_SIZE; + /// let pages = VVec::::with_capacity(size, GFP_KERNEL)?; + /// + /// let sgt = KBox::pin_init(SGTable::new( + /// dev, + /// pages, + /// dma::DataDirection::ToDevice, + /// GFP_KERNEL, + /// ), GFP_KERNEL)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn new( + dev: &Device, + pages: P, + dir: dma::DataDirection, + flags: alloc::Flags, + ) -> impl PinInit + '_ { + try_pin_init!(Self { + inner <- Owned::new(dev, pages, dir, flags)? + }) + } +} + +impl

Deref for SGTable> { + type Target = SGTable; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - `self.inner.sgt.as_raw()` is a valid pointer to a `struct sg_table` for the entire + // lifetime of `self`. + // - The backing `struct sg_table` is not modified for the entire lifetime of `self`. + unsafe { SGTable::from_raw(self.inner.sgt.as_raw()) } + } +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::Borrowed {} + impl

Sealed for super::Owned

{} +} + +/// An [`Iterator`] over the DMA mapped [`SGEntry`] items of an [`SGTable`]. +/// +/// Note that the existence of an [`SGTableIter`] does not guarantee that the [`SGEntry`] items +/// actually remain DMA mapped; they are prone to be unmapped on device unbind. +pub struct SGTableIter<'a> { + pos: Option<&'a SGEntry>, + /// The number of DMA mapped entries in a `struct sg_table`. + nents: c_uint, +} + +impl<'a> Iterator for SGTableIter<'a> { + type Item = &'a SGEntry; + + fn next(&mut self) -> Option { + let entry = self.pos?; + self.nents = self.nents.saturating_sub(1); + + // SAFETY: `entry.as_raw()` is a valid pointer to a `struct scatterlist`. + let next = unsafe { bindings::sg_next(entry.as_raw()) }; + + self.pos = (!next.is_null() && self.nents > 0).then(|| { + // SAFETY: If `next` is not NULL, `sg_next()` guarantees to return a valid pointer to + // the next `struct scatterlist`. + unsafe { SGEntry::from_raw(next) } + }); + + Some(entry) + } +} diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs new file mode 100644 index 000000000..9d2716952 --- /dev/null +++ b/rust/kernel/security.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Linux Security Modules (LSM). +//! +//! C header: [`include/linux/security.h`](srctree/include/linux/security.h). + +use crate::{ + bindings, + cred::Credential, + error::{to_result, Result}, + fs::File, +}; + +/// Calls the security modules to determine if the given task can become the manager of a binder +/// context. +#[inline] +pub fn binder_set_context_mgr(mgr: &Credential) -> Result { + // SAFETY: `mrg.0` is valid because the shared reference guarantees a nonzero refcount. + to_result(unsafe { bindings::security_binder_set_context_mgr(mgr.as_ptr()) }) +} + +/// Calls the security modules to determine if binder transactions are allowed from task `from` to +/// task `to`. +#[inline] +pub fn binder_transaction(from: &Credential, to: &Credential) -> Result { + // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts. + to_result(unsafe { bindings::security_binder_transaction(from.as_ptr(), to.as_ptr()) }) +} + +/// Calls the security modules to determine if task `from` is allowed to send binder objects +/// (owned by itself or other processes) to task `to` through a binder transaction. +#[inline] +pub fn binder_transfer_binder(from: &Credential, to: &Credential) -> Result { + // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts. + to_result(unsafe { bindings::security_binder_transfer_binder(from.as_ptr(), to.as_ptr()) }) +} + +/// Calls the security modules to determine if task `from` is allowed to send the given file to +/// task `to` (which would get its own file descriptor) through a binder transaction. +#[inline] +pub fn binder_transfer_file(from: &Credential, to: &Credential, file: &File) -> Result { + // SAFETY: `from`, `to` and `file` are valid because the shared references guarantee nonzero + // refcounts. + to_result(unsafe { + bindings::security_binder_transfer_file(from.as_ptr(), to.as_ptr(), file.as_ptr()) + }) +} + +/// A security context string. +/// +/// # Invariants +/// +/// The `ctx` field corresponds to a valid security context as returned by a successful call to +/// `security_secid_to_secctx`, that has not yet been released by `security_release_secctx`. +pub struct SecurityCtx { + ctx: bindings::lsm_context, +} + +impl SecurityCtx { + /// Get the security context given its id. + #[inline] + pub fn from_secid(secid: u32) -> Result { + // SAFETY: `struct lsm_context` can be initialized to all zeros. + let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() }; + + // SAFETY: Just a C FFI call. The pointer is valid for writes. + to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut ctx) })?; + + // INVARIANT: If the above call did not fail, then we have a valid security context. + Ok(Self { ctx }) + } + + /// Returns whether the security context is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.ctx.len == 0 + } + + /// Returns the length of this security context. + #[inline] + pub fn len(&self) -> usize { + self.ctx.len as usize + } + + /// Returns the bytes for this security context. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + let ptr = self.ctx.context; + if ptr.is_null() { + debug_assert_eq!(self.len(), 0); + // We can't pass a null pointer to `slice::from_raw_parts` even if the length is zero. + return &[]; + } + + // SAFETY: The call to `security_secid_to_secctx` guarantees that the pointer is valid for + // `self.len()` bytes. Furthermore, if the length is zero, then we have ensured that the + // pointer is not null. + unsafe { core::slice::from_raw_parts(ptr.cast(), self.len()) } + } +} + +impl Drop for SecurityCtx { + #[inline] + fn drop(&mut self) { + // SAFETY: By the invariant of `Self`, this releases an lsm context that came from a + // successful call to `security_secid_to_secctx` and has not yet been released. + unsafe { bindings::security_release_secctx(&mut self.ctx) }; + } +} diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs new file mode 100644 index 000000000..518265558 --- /dev/null +++ b/rust/kernel/seq_file.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Seq file bindings. +//! +//! C header: [`include/linux/seq_file.h`](srctree/include/linux/seq_file.h) + +use crate::{bindings, fmt, str::CStrExt as _, types::NotThreadSafe, types::Opaque}; + +/// A utility for generating the contents of a seq file. +#[repr(transparent)] +pub struct SeqFile { + inner: Opaque, + _not_send: NotThreadSafe, +} + +impl SeqFile { + /// Creates a new [`SeqFile`] from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of `'a` the following is satisfied: + /// * The pointer points at a valid `struct seq_file`. + /// * The `struct seq_file` is not accessed from any other thread. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::seq_file) -> &'a SeqFile { + // SAFETY: The caller ensures that the reference is valid for 'a. There's no way to trigger + // a data race by using the `&SeqFile` since this is the only thread accessing the seq_file. + // + // CAST: The layout of `struct seq_file` and `SeqFile` is compatible. + unsafe { &*ptr.cast() } + } + + /// Used by the [`seq_print`] macro. + #[inline] + pub fn call_printf(&self, args: fmt::Arguments<'_>) { + // SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`. + unsafe { + bindings::seq_printf( + self.inner.get(), + c"%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } + } +} + +/// Write to a [`SeqFile`] with the ordinary Rust formatting syntax. +#[macro_export] +macro_rules! seq_print { + ($m:expr, $($arg:tt)+) => ( + $m.call_printf($crate::prelude::fmt!($($arg)+)) + ); +} +pub use seq_print; diff --git a/rust/kernel/serdev.rs b/rust/kernel/serdev.rs new file mode 100644 index 000000000..17ca504b7 --- /dev/null +++ b/rust/kernel/serdev.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the serial device bus. +//! +//! C header: [`include/linux/serdev.h`](srctree/include/linux/serdev.h) + +use crate::{ + acpi, + device, + driver, + error::{ + from_result, + to_result, + VTABLE_DEFAULT_ERROR, // + }, + new_mutex, + of, + prelude::*, + sync::{ + aref::AlwaysRefCounted, + Mutex, // + }, + time::Jiffies, + types::{ + Opaque, + ScopeGuard, // + }, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{offset_of, MaybeUninit}, + ptr::NonNull, // +}; + +/// Parity bit to use with a serial device. +#[repr(u32)] +pub enum Parity { + /// No parity bit. + None = bindings::serdev_parity_SERDEV_PARITY_NONE, + /// Even partiy. + Even = bindings::serdev_parity_SERDEV_PARITY_EVEN, + /// Odd parity. + Odd = bindings::serdev_parity_SERDEV_PARITY_ODD, +} + +/// An adapter for the registration of serial device bus device drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::serdev_device_driver` is a C type declared as `repr(C)`. +// - `PrivateData<'bound, T>` is the type of the driver's device private data. +// - `struct serdev_device_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::serdev_device_driver; + type DriverData<'bound> = PrivateData<'bound, T>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + sdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct serdev_device_driver` on initialization. + unsafe { + (*sdrv.get()).driver.name = name.as_char_ptr(); + (*sdrv.get()).probe = Some(Self::probe_callback); + (*sdrv.get()).remove = Some(Self::remove_callback); + (*sdrv.get()).driver.of_match_table = of_table; + (*sdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::__serdev_device_driver_register(sdrv.get(), module.as_ptr()) }) + } + + unsafe fn unregister(sdrv: &Opaque) { + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::serdev_device_driver_unregister(sdrv.get()) }; + } +} + +#[doc(hidden)] +#[pin_data(PinnedDrop)] +pub struct PrivateData<'bound, T: Driver> { + sdev: &'bound Device, + #[pin] + driver: UnsafeCell>>, + open: UnsafeCell, + /// Whether `receive_buf_callback` is allowed to call `Driver::receive`. + /// + /// If locked, the receive_buf_callback will be blocked on data reception. + /// This is the case while the driver is being probed or while [`PrivateData`] is being dropped. + /// This is necessary, because we need to open the serdev device before the driver has been + /// probed in order to allow it to be configured, which allows `receive_buf_callback` to be + /// called. Thus we need to block data until probe completes and the driver data becomes + /// initialized. + /// + /// If unlocked and true, the receive_buf_callback will forward the data to + /// `Driver::receive`. This is the normal state of operation. + /// + /// If unlocked and false, the receive_buf_callback will throw away the data. + /// This is only the case, if the serdev device is open and + /// - the driver returned an error in probe + /// or + /// - the driver data already has been dropped, because it was unbound. + #[pin] + active: Mutex, +} + +#[pinned_drop] +impl PinnedDrop for PrivateData<'_, T> { + fn drop(self: Pin<&mut Self>) { + let mut active = self.active.lock(); + if *active { + // SAFETY: + // - We have exclusive access to `self.driver`. + // - `self.driver` is guaranteed to be initialized. + unsafe { (*self.driver.get()).assume_init_drop() }; + *active = false; + } + drop(active); + + // SAFETY: We have exclusive access to `self.open`. + if unsafe { *self.open.get() } { + // SAFETY: `self.sdev.as_raw()` is guaranteed to be a pointer to a valid + // `struct serdev_device`. + unsafe { bindings::serdev_device_close(self.sdev.as_raw()) }; + } + } +} + +impl Adapter { + const OPS: &'static bindings::serdev_device_ops = &bindings::serdev_device_ops { + receive_buf: if T::HAS_RECEIVE { + Some(Self::receive_buf_callback) + } else { + None + }, + write_wakeup: Some(bindings::serdev_device_write_wakeup), + }; + + extern "C" fn probe_callback(sdev: *mut bindings::serdev_device) -> kernel::ffi::c_int { + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer to + // a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `probe_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + // SAFETY: `sdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(sdev.as_ref()) }; + + from_result(|| { + sdev.as_ref().set_drvdata(try_pin_init!(PrivateData:: { + sdev: &**sdev, + driver: MaybeUninit::>::zeroed().into(), + open: false.into(), + active <- new_mutex!(false), + }))?; + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let private_data = ScopeGuard::new_with_data(private_data, |_| { + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + drop(unsafe { sdev.as_ref().drvdata_obtain::>() }); + }); + let mut active = private_data.active.lock(); + + // SAFETY: `sdev.as_raw()` is guaranteed to be a valid pointer to `serdev_device`. + unsafe { bindings::serdev_device_set_client_ops(sdev.as_raw(), Self::OPS) }; + + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer + // to a `serdev_device`. + to_result(unsafe { bindings::serdev_device_open(sdev.as_raw()) })?; + + // SAFETY: We have exclusive access to `private_data.open`. + unsafe { *private_data.open.get() = true }; + + let data = T::probe(sdev, info); + + // SAFETY: We have exclusive access to `private_data.driver`. + let driver = unsafe { &mut *private_data.driver.get() }; + // SAFETY: + // - `driver.as_mut_ptr()` is a valid pointer to uninitialized data. + // - `private_data.driver` is pinned. + let result = unsafe { pin_init::raw_try_init(driver.as_mut_ptr(), data) }; + + *active = result.is_ok(); + + drop(active); + + result.map(|()| { + private_data.dismiss(); + 0 + }) + }) + } + + extern "C" fn remove_callback(sdev: *mut bindings::serdev_device) { + // SAFETY: The serial device bus only ever calls the remove callback with a valid pointer + // to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `remove_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `remove_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + T::unbind(sdev, data_pinned); + } + + extern "C" fn receive_buf_callback( + sdev: *mut bindings::serdev_device, + buf: *const u8, + length: usize, + ) -> usize { + // SAFETY: The serial device bus only ever calls the receive buf callback with a valid + // pointer to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `receive_buf_callback()`. + let sdev = unsafe { &*sdev.cast::>() }; + + // SAFETY: `receive_buf_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let active = private_data.active.lock(); + + if !*active { + return length; + } + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `receive_buf_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + // SAFETY: `buf` is guaranteed to be non-null and has the size of `length`. + let buf = unsafe { core::slice::from_raw_parts(buf, length) }; + + T::receive(sdev, data_pinned, buf) + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single serial device bus device driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_serdev_device_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_serdev_device_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::serdev::Adapter, { $($f)* }); + }; +} + +/// The serial device bus device driver trait. +/// +/// Drivers must implement this trait in order to get a serial device bus device driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// acpi, +/// bindings, +/// device::{ +/// Bound, +/// Core, // +/// }, +/// of, +/// serdev, // +/// }; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// #[vtable] +/// impl serdev::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// sdev: &'bound serdev::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// sdev.set_baudrate(115200); +/// sdev.write_all(b"Hello\n", 0)?; +/// Ok(MyDriver) +/// } +/// } +///``` +#[vtable] +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + Sync + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// Serial device bus device driver probe. + /// + /// Called when a new serial device bus device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + sdev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// Serial device bus device driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(sdev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (sdev, this); + } + + /// Serial device bus device data receive callback. + /// + /// Called when data got received from device. + /// + /// Returns the number of bytes accepted. + fn receive<'bound>( + sdev: &'bound Device, + this: Pin<&Self::Data<'bound>>, + data: &[u8], + ) -> usize { + let _ = (sdev, this, data); + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// The serial device bus device representation. +/// +/// This structure represents the Rust abstraction for a C `struct serdev_device`. The +/// implementation abstracts the usage of an already existing C `struct serdev_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct serdev_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::serdev_device { + self.0.get() + } +} + +impl Device { + /// Set the baudrate in bits per second. + /// + /// Common baudrates are 115200, 9600, 19200, 57600, 4800. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_baudrate(&self, speed: u32) -> Result<(), u32> { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + let ret = unsafe { bindings::serdev_device_set_baudrate(self.as_raw(), speed) }; + if ret == speed { + Ok(()) + } else { + Err(ret) + } + } + + /// Set if flow control should be enabled. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_flow_control(&self, enable: bool) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_set_flow_control(self.as_raw(), enable) }; + } + + /// Set parity to use. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_parity(&self, parity: Parity) -> Result { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + to_result(unsafe { bindings::serdev_device_set_parity(self.as_raw(), parity as u32) }) + } + + /// Write data to the serial device until the controller has accepted all the data or has + /// been interrupted by a timeout or signal. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Use a timeout of 0 to wait indefinitely. + /// + /// Returns the number of bytes written (less than `data.len()` if interrupted). + /// [`kernel::error::code::ETIMEDOUT`] or [`kernel::error::code::ERESTARTSYS`] if interrupted + /// before any bytes were written. [`kernel::error::code::EINVAL`] if `data.len() > i32::MAX`. + #[inline] + pub fn write_all(&self, data: &[u8], timeout: Jiffies) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = unsafe { + bindings::serdev_device_write( + self.as_raw(), + data.as_ptr(), + data.len(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + // CAST: negative return values are guaranteed to be between `-MAX_ERRNO` and `-1`, + // which always fit into a `i32`. + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Write data to the serial device. + /// + /// If you want to write until the controller has accepted all the data, use + /// [`Device::write_all`]. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Returns the number of bytes written (less than `data.len()` if not enough room in the + /// write buffer). + #[inline] + pub fn write(&self, data: &[u8]) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = + unsafe { bindings::serdev_device_write_buf(self.as_raw(), data.as_ptr(), data.len()) }; + + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Send data to the serial device immediately. + /// + /// Note that this doesn't guarantee that the data has been transmitted. + /// Use [`Device::wait_until_sent`] for this purpose. + #[inline] + pub fn write_flush(&self) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_write_flush(self.as_raw()) }; + } + + /// Wait for the data to be sent. + /// + /// After this function, the write buffer of the controller should be empty or the timeout + /// elapsed. + /// + /// Use a timeout of 0 to wait indefinitely. + #[inline] + pub fn wait_until_sent(&self, timeout: Jiffies) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { + bindings::serdev_device_wait_until_sent( + self.as_raw(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + } +} + +// SAFETY: `serdev::Device` is a transparent wrapper of `struct serdev_device`. +// The offset is guaranteed to point to a valid device field inside `serdev::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::serdev_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + self.as_ref().inc_ref(); + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::serdev_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct serdev_device`. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct serdev_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs new file mode 100644 index 000000000..521b2b38b --- /dev/null +++ b/rust/kernel/sizes.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Commonly used sizes. +//! +//! C headers: [`include/linux/sizes.h`](srctree/include/linux/sizes.h). +//! +//! The top-level `SZ_*` constants are [`usize`]-typed, for use in kernel page +//! arithmetic and similar CPU-side work. +//! +//! The [`SizeConstants`] trait provides the same constants as associated constants +//! on [`u32`], [`u64`], and [`usize`], for use in device address spaces where +//! the address width depends on the hardware. Device drivers frequently need +//! these constants as [`u64`] (or [`u32`]) rather than [`usize`], because +//! device address spaces are sized independently of the CPU pointer width. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! page::PAGE_SIZE, +//! sizes::{ +//! SizeConstants, +//! SZ_1M, // +//! }, // +//! }; +//! +//! // Module-level constants continue to work without a type qualifier. +//! let num_pages_in_1m = SZ_1M / PAGE_SIZE; +//! +//! // Trait associated constants require a type qualifier. +//! let heap_size = 14 * u64::SZ_1M; +//! let small = u32::SZ_4K; +//! ``` + +macro_rules! define_sizes { + ($($type:ty),* $(,)?) => { + define_sizes!(@internal [$($type),*] + /// `0x0000_0400`. + SZ_1K, + /// `0x0000_0800`. + SZ_2K, + /// `0x0000_1000`. + SZ_4K, + /// `0x0000_2000`. + SZ_8K, + /// `0x0000_4000`. + SZ_16K, + /// `0x0000_8000`. + SZ_32K, + /// `0x0001_0000`. + SZ_64K, + /// `0x0002_0000`. + SZ_128K, + /// `0x0004_0000`. + SZ_256K, + /// `0x0008_0000`. + SZ_512K, + /// `0x0010_0000`. + SZ_1M, + /// `0x0020_0000`. + SZ_2M, + /// `0x0040_0000`. + SZ_4M, + /// `0x0080_0000`. + SZ_8M, + /// `0x0100_0000`. + SZ_16M, + /// `0x0200_0000`. + SZ_32M, + /// `0x0400_0000`. + SZ_64M, + /// `0x0800_0000`. + SZ_128M, + /// `0x1000_0000`. + SZ_256M, + /// `0x2000_0000`. + SZ_512M, + /// `0x4000_0000`. + SZ_1G, + /// `0x8000_0000`. + SZ_2G, + ); + }; + + (@internal [$($type:ty),*] $($names_and_metas:tt)*) => { + define_sizes!(@consts_and_trait $($names_and_metas)*); + define_sizes!(@impls [$($type),*] $($names_and_metas)*); + }; + + (@consts_and_trait $($(#[$meta:meta])* $name:ident,)*) => { + $( + $(#[$meta])* + pub const $name: usize = bindings::$name as usize; + )* + + /// Size constants for device address spaces. + /// + /// Implemented for [`u32`], [`u64`], and [`usize`] so drivers can + /// choose the width that matches their hardware. All `SZ_*` values fit + /// in a [`u32`], so all implementations are lossless. + /// + /// # Examples + /// + /// ``` + /// use kernel::sizes::SizeConstants; + /// + /// let gpu_heap = 14 * u64::SZ_1M; + /// let mmio_window = u32::SZ_16M; + /// ``` + pub trait SizeConstants { + $( + $(#[$meta])* + const $name: Self; + )* + } + }; + + (@impls [] $($(#[$meta:meta])* $name:ident,)*) => {}; + + (@impls [$first:ty $(, $rest:ty)*] $($(#[$meta:meta])* $name:ident,)*) => { + impl SizeConstants for $first { + $( + const $name: Self = { + assert!((self::$name as u128) <= (<$first>::MAX as u128)); + self::$name as $first + }; + )* + } + + define_sizes!(@impls [$($rest),*] $($(#[$meta])* $name,)*); + }; +} + +define_sizes!(u32, u64, usize); diff --git a/rust/kernel/soc.rs b/rust/kernel/soc.rs new file mode 100644 index 000000000..0d6a36c83 --- /dev/null +++ b/rust/kernel/soc.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! SoC Driver Abstraction. +//! +//! C header: [`include/linux/sys_soc.h`](srctree/include/linux/sys_soc.h) + +use crate::{ + bindings, + error, + prelude::*, + str::CString, + types::Opaque, // +}; +use core::ptr::NonNull; + +/// Attributes for a SoC device. +/// +/// These are both exported to userspace under /sys/devices/socX and provided to other drivers to +/// match against via `soc_device_match` (not yet available in Rust) to enable quirks or +/// device-specific support where necessary. +/// +/// All fields are freeform - they have no specific formatting, just defined meanings. +/// For example, the [`machine`](`Attributes::machine`) field could be "DB8500" or +/// "Qualcomm Technologies, Inc. SM8560 HDK", but regardless it should identify a board or product. +pub struct Attributes { + /// Should generally be a board ID or product ID. Examples + /// include DB8500 (ST-Ericsson) or "Qualcomm Technologies, inc. SM8560 HDK". + /// + /// If this field is not populated, the SoC infrastructure will try to populate it from + /// `/model` in the device tree. + pub machine: Option, + /// The broader class this SoC belongs to. Examples include ux500 + /// (for DB8500) or Snapdragon (for SM8650). + /// + /// On chips with ARM firmware supporting SMCCC v1.2+, this may be a JEDEC JEP106 manufacturer + /// identification. + pub family: Option, + /// The manufacturing revision of the part. Frequently this is MAJOR.MINOR, but not always. + pub revision: Option, + /// Serial Number - uniquely identifies a specific SoC. If present, should be unique (buying a + /// replacement part should change it if present). This field cannot be matched on and is + /// solely present to export through /sys. + pub serial_number: Option, + /// SoC ID - identifies a specific SoC kind in question, sometimes more specifically than + /// `machine` if the same SoC is used in multiple products. Some devices use this to specify a + /// SoC name, e.g. "I.MX??", and others just print an ID number (e.g. Tegra and Qualcomm). + /// + /// On chips with ARM firmware supporting SMCCC v1.2+, this may be a JEDEC JEP106 manufacturer + /// identification (the family value) followed by a colon and then a 4-digit ID value. + pub soc_id: Option, +} + +struct BuiltAttributes { + // While `inner` has pointers to `_backing`, it is to the interior of the `CStrings`, not + // `backing` itself, so it does not need to be pinned. + _backing: Attributes, + // `Opaque` makes us `!Unpin`, as the registration holds a pointer to `inner` when used. + inner: Opaque, +} + +fn cstring_to_c(mcs: &Option) -> *const kernel::ffi::c_char { + mcs.as_ref() + .map(|cs| cs.as_char_ptr()) + .unwrap_or(core::ptr::null()) +} + +impl BuiltAttributes { + fn as_mut_ptr(&self) -> *mut bindings::soc_device_attribute { + self.inner.get() + } +} + +impl Attributes { + fn build(self) -> BuiltAttributes { + BuiltAttributes { + inner: Opaque::new(bindings::soc_device_attribute { + machine: cstring_to_c(&self.machine), + family: cstring_to_c(&self.family), + revision: cstring_to_c(&self.revision), + serial_number: cstring_to_c(&self.serial_number), + soc_id: cstring_to_c(&self.soc_id), + data: core::ptr::null(), + custom_attr_group: core::ptr::null(), + }), + _backing: self, + } + } +} + +#[pin_data(PinnedDrop)] +/// Registration handle for your soc_dev. If you let it go out of scope, your soc_dev will be +/// unregistered. +pub struct Registration { + #[pin] + attr: BuiltAttributes, + soc_dev: NonNull, +} + +// SAFETY: We provide no operations through `&Registration`. +unsafe impl Sync for Registration {} + +// SAFETY: All pointers are normal allocations, not thread-specific. +unsafe impl Send for Registration {} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Device always contains a live pointer to a soc_device that can be unregistered + unsafe { bindings::soc_device_unregister(self.soc_dev.as_ptr()) } + } +} + +impl Registration { + /// Register a new SoC device + pub fn new(attr: Attributes) -> impl PinInit { + try_pin_init!(Self { + attr: attr.build(), + soc_dev: { + // SAFETY: + // * The struct provided through attr is backed by pinned data next to it, + // so as long as attr lives, the strings pointed to by the struct will too. + // * `attr` is pinned, so the pinned data won't move. + // * If it returns a device, and so others may try to read this data, by + // caller invariant, `attr` won't be released until the device is. + let raw_soc = error::from_err_ptr(unsafe { + bindings::soc_device_register(attr.as_mut_ptr()) + })?; + + NonNull::new(raw_soc).ok_or(EINVAL)? + }, + }? Error) + } +} diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs new file mode 100644 index 000000000..abbab5050 --- /dev/null +++ b/rust/kernel/std_vendor.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . + +/// [`std::dbg`], but using [`pr_info`] instead of [`eprintln`]. +/// +/// Prints and returns the value of a given expression for quick and dirty +/// debugging. +/// +/// An example: +/// +/// ```rust +/// let a = 2; +/// # #[expect(clippy::disallowed_macros)] +/// let b = dbg!(a * 2) + 1; +/// // ^-- prints: [src/main.rs:3:9] a * 2 = 4 +/// assert_eq!(b, 5); +/// ``` +/// +/// The macro works by using the `Debug` implementation of the type of +/// the given expression to print the value with [`printk`] along with the +/// source location of the macro invocation as well as the source code +/// of the expression. +/// +/// Invoking the macro on an expression moves and takes ownership of it +/// before returning the evaluated expression unchanged. If the type +/// of the expression does not implement `Copy` and you don't want +/// to give up ownership, you can instead borrow with `dbg!(&expr)` +/// for some expression `expr`. +/// +/// The `dbg!` macro works exactly the same in release builds. +/// This is useful when debugging issues that only occur in release +/// builds or when debugging in release mode is significantly faster. +/// +/// Note that the macro is intended as a temporary debugging tool to be +/// used during development. Therefore, avoid committing `dbg!` macro +/// invocations into the kernel tree. +/// +/// For debug output that is intended to be kept in the kernel tree, +/// use [`pr_debug`] and similar facilities instead. +/// +/// # Stability +/// +/// The exact output printed by this macro should not be relied upon +/// and is subject to future changes. +/// +/// # Further examples +/// +/// With a method call: +/// +/// ```rust +/// # #[expect(clippy::disallowed_macros)] +/// fn foo(n: usize) { +/// if dbg!(n.checked_sub(4)).is_some() { +/// // ... +/// } +/// } +/// +/// foo(3) +/// ``` +/// +/// This prints to the kernel log: +/// +/// ```text,ignore +/// [src/main.rs:3:8] n.checked_sub(4) = None +/// ``` +/// +/// Naive factorial implementation: +/// +/// ```rust +/// # #![expect(clippy::disallowed_macros)] +/// fn factorial(n: u32) -> u32 { +/// if dbg!(n <= 1) { +/// dbg!(1) +/// } else { +/// dbg!(n * factorial(n - 1)) +/// } +/// } +/// +/// dbg!(factorial(4)); +/// ``` +/// +/// This prints to the kernel log: +/// +/// ```text,ignore +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = true +/// [src/main.rs:4:9] 1 = 1 +/// [src/main.rs:5:9] n * factorial(n - 1) = 2 +/// [src/main.rs:5:9] n * factorial(n - 1) = 6 +/// [src/main.rs:5:9] n * factorial(n - 1) = 24 +/// [src/main.rs:11:1] factorial(4) = 24 +/// ``` +/// +/// The `dbg!(..)` macro moves the input: +/// +/// ```ignore +/// /// A wrapper around `usize` which importantly is not Copyable. +/// #[derive(Debug)] +/// struct NoCopy(usize); +/// +/// let a = NoCopy(42); +/// let _ = dbg!(a); // <-- `a` is moved here. +/// let _ = dbg!(a); // <-- `a` is moved again; error! +/// ``` +/// +/// You can also use `dbg!()` without a value to just print the +/// file and line whenever it's reached. +/// +/// Finally, if you want to `dbg!(..)` multiple values, it will treat them as +/// a tuple (and return it, too): +/// +/// ``` +/// # #![expect(clippy::disallowed_macros)] +/// assert_eq!(dbg!(1usize, 2u32), (1, 2)); +/// ``` +/// +/// However, a single argument with a trailing comma will still not be treated +/// as a tuple, following the convention of ignoring trailing commas in macro +/// invocations. You can use a 1-tuple directly if you need one: +/// +/// ``` +/// # #![expect(clippy::disallowed_macros)] +/// assert_eq!(1, dbg!(1u32,)); // trailing comma ignored +/// assert_eq!((1,), dbg!((1u32,))); // 1-tuple +/// ``` +/// +/// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html +/// [`eprintln`]: https://doc.rust-lang.org/std/macro.eprintln.html +/// [`printk`]: https://docs.kernel.org/core-api/printk-basics.html +/// [`pr_info`]: crate::pr_info! +/// [`pr_debug`]: crate::pr_debug! +#[macro_export] +macro_rules! dbg { + // NOTE: We cannot use `concat!` to make a static string as a format argument + // of `pr_info!` because `file!` could contain a `{` or + // `$val` expression could be a block (`{ .. }`), in which case the `pr_info!` + // will be malformed. + () => { + $crate::pr_info!("[{}:{}:{}]\n", ::core::file!(), ::core::line!(), ::core::column!()) + }; + ($val:expr $(,)?) => { + // Use of `match` here is intentional because it affects the lifetimes + // of temporaries - + match $val { + tmp => { + $crate::pr_info!("[{}:{}:{}] {} = {:#?}\n", + ::core::file!(), ::core::line!(), ::core::column!(), + ::core::stringify!($val), &tmp); + tmp + } + } + }; + ($($val:expr),+ $(,)?) => { + ($($crate::dbg!($val)),+,) + }; +} diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs new file mode 100644 index 000000000..b3caa9a1c --- /dev/null +++ b/rust/kernel/str.rs @@ -0,0 +1,918 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! String representations. + +use crate::{ + alloc::{ + AllocError, + KVec, // + }, + error::{ + to_result, + Result, // + }, + fmt::{ + self, + Write, // + }, + prelude::*, // +}; +use core::{ + marker::PhantomData, + ops::{ + Deref, + DerefMut, + Index, // + }, // +}; + +pub use crate::prelude::CStr; + +pub mod parse_int; + +/// Byte string without UTF-8 validity guarantee. +#[repr(transparent)] +pub struct BStr([u8]); + +impl BStr { + /// Returns the length of this string. + #[inline] + pub const fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the string is empty. + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a [`BStr`] from a `[u8]`. + #[inline] + pub const fn from_bytes(bytes: &[u8]) -> &Self { + // SAFETY: `BStr` is transparent to `[u8]`. + unsafe { &*(core::ptr::from_ref(bytes) as *const BStr) } + } + + /// Strip a prefix from `self`. Delegates to [`slice::strip_prefix`]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::b_str; + /// assert_eq!(Some(b_str!("bar")), b_str!("foobar").strip_prefix(b_str!("foo"))); + /// assert_eq!(None, b_str!("foobar").strip_prefix(b_str!("bar"))); + /// assert_eq!(Some(b_str!("foobar")), b_str!("foobar").strip_prefix(b_str!(""))); + /// assert_eq!(Some(b_str!("")), b_str!("foobar").strip_prefix(b_str!("foobar"))); + /// ``` + pub fn strip_prefix(&self, pattern: impl AsRef) -> Option<&BStr> { + self.deref() + .strip_prefix(pattern.as_ref().deref()) + .map(Self::from_bytes) + } +} + +impl fmt::Display for BStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; + /// let ascii = b_str!("Hello, BStr!"); + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes(), "Hello, BStr!".as_bytes()); + /// + /// let non_ascii = b_str!("🦀"); + /// let s = CString::try_from_fmt(fmt!("{non_ascii}"))?; + /// assert_eq!(s.to_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{b:02x}")?, + } + } + Ok(()) + } +} + +impl fmt::Debug for BStr { + /// Formats printable ASCII characters with a double quote on either end, + /// escaping the rest. + /// + /// ``` + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; + /// // Embedded double quotes are escaped. + /// let ascii = b_str!("Hello, \"BStr\"!"); + /// let s = CString::try_from_fmt(fmt!("{ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); + /// + /// let non_ascii = b_str!("😺"); + /// let s = CString::try_from_fmt(fmt!("{non_ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // String escape characters. + b'\"' => f.write_str("\\\"")?, + b'\\' => f.write_str("\\\\")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{b:02x}")?, + } + } + f.write_char('"') + } +} + +impl Deref for BStr { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for BStr { + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl Index for BStr +where + [u8]: Index, +{ + type Output = Self; + + fn index(&self, index: Idx) -> &Self::Output { + BStr::from_bytes(&self.0[index]) + } +} + +impl AsRef for [u8] { + fn as_ref(&self) -> &BStr { + BStr::from_bytes(self) + } +} + +impl AsRef for BStr { + fn as_ref(&self) -> &BStr { + self + } +} + +/// Creates a new [`BStr`] from a string literal. +/// +/// `b_str!` converts the supplied string literal to byte string, so non-ASCII +/// characters can be included. +/// +/// # Examples +/// +/// ``` +/// # use kernel::b_str; +/// # use kernel::str::BStr; +/// const MY_BSTR: &BStr = b_str!("My awesome BStr!"); +/// ``` +#[macro_export] +macro_rules! b_str { + ($str:literal) => {{ + const S: &'static str = $str; + const C: &'static $crate::str::BStr = $crate::str::BStr::from_bytes(S.as_bytes()); + C + }}; +} + +/// Returns a C pointer to the string. +// It is a free function rather than a method on an extension trait because: +// +// - error[E0379]: functions in trait impls cannot be declared const +#[inline] +#[expect(clippy::disallowed_methods, reason = "internal implementation")] +pub const fn as_char_ptr_in_const_context(c_str: &CStr) -> *const c_char { + c_str.as_ptr().cast() +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::CStr {} +} + +/// Extensions to [`CStr`]. +pub trait CStrExt: private::Sealed { + /// Wraps a raw C string pointer. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a `NUL`-terminated C string, and it must + /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr` + /// must not be mutated. + // This function exists to paper over the fact that `CStr::from_ptr` takes a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self; + + /// Creates a mutable [`CStr`] from a `[u8]` without performing any + /// additional checks. + /// + /// # Safety + /// + /// `bytes` *must* end with a `NUL` byte, and should only have a single + /// `NUL` byte (or the string will be truncated). + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self; + + /// Returns a C pointer to the string. + // This function exists to paper over the fact that `CStr::as_ptr` returns a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + fn as_char_ptr(&self) -> *const c_char; + + /// Convert this [`CStr`] into a [`CString`] by allocating memory and + /// copying over the string data. + fn to_cstring(&self) -> Result; + + /// Converts this [`CStr`] to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`to_ascii_lowercase()`]. + /// + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase + fn make_ascii_lowercase(&mut self); + + /// Converts this [`CStr`] to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`to_ascii_uppercase()`]. + /// + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase + fn make_ascii_uppercase(&mut self); + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// + /// [`make_ascii_lowercase`]: str::make_ascii_lowercase + fn to_ascii_lowercase(&self) -> Result; + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// + /// [`make_ascii_uppercase`]: str::make_ascii_uppercase + fn to_ascii_uppercase(&self) -> Result; +} + +impl fmt::Display for CStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::prelude::fmt; + /// # use kernel::str::CStr; + /// # use kernel::str::CString; + /// let penguin = c"🐧"; + /// let s = CString::try_from_fmt(fmt!("{penguin}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); + /// + /// let ascii = c"so \"cool\""; + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "so \"cool\"\0".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &c in self.to_bytes() { + if (0x20..0x7f).contains(&c) { + // Printable character. + f.write_char(c as char)?; + } else { + write!(f, "\\x{c:02x}")?; + } + } + Ok(()) + } +} + +/// Converts a mutable C string to a mutable byte slice. +/// +/// # Safety +/// +/// The caller must ensure that the slice ends in a NUL byte and contains no other NUL bytes before +/// the borrow ends and the underlying [`CStr`] is used. +unsafe fn to_bytes_mut(s: &mut CStr) -> &mut [u8] { + // SAFETY: the cast from `&CStr` to `&[u8]` is safe since `CStr` has the same layout as `&[u8]` + // (this is technically not guaranteed, but we rely on it here). The pointer dereference is + // safe since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(s) as *mut [u8]) } +} + +impl CStrExt for CStr { + #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { + // SAFETY: The safety preconditions are the same as for `CStr::from_ptr`. + unsafe { CStr::from_ptr(ptr.cast()) } + } + + #[inline] + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self { + // SAFETY: the cast from `&[u8]` to `&CStr` is safe since the properties of `bytes` are + // guaranteed by the safety precondition and `CStr` has the same layout as `&[u8]` (this is + // technically not guaranteed, but we rely on it here). The pointer dereference is safe + // since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(bytes) as *mut CStr) } + } + + #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] + fn as_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } + + fn to_cstring(&self) -> Result { + CString::try_from(self) + } + + fn make_ascii_lowercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_lowercase(); + } + + fn make_ascii_uppercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_uppercase(); + } + + fn to_ascii_lowercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_lowercase(); + + Ok(s) + } + + fn to_ascii_uppercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_uppercase(); + + Ok(s) + } +} + +impl AsRef for CStr { + #[inline] + fn as_ref(&self) -> &BStr { + BStr::from_bytes(self.to_bytes()) + } +} + +/// Creates a new [`CStr`] at compile time. +/// +/// Rust supports C string literals since Rust 1.77, and they should be used instead of this macro +/// where possible. This macro exists to allow static *non-literal* C strings to be created at +/// compile time. This is most often used in other macros. +/// +/// # Panics +/// +/// This macro panics if the operand contains an interior `NUL` byte. +/// +/// # Examples +/// +/// ``` +/// # use kernel::c_str; +/// # use kernel::str::CStr; +/// // This is allowed, but `c"literal"` should be preferred for literals. +/// const BAD: &CStr = c_str!("literal"); +/// +/// // `c_str!` is still needed for static non-literal C strings. +/// const GOOD: &CStr = c_str!(concat!(file!(), ":", line!(), ": My CStr!")); +/// ``` +#[macro_export] +macro_rules! c_str { + // NB: We could write `($str:lit) => compile_error!("use a C string literal instead");` here but + // that would trigger when the literal is at the top of several macro expansions. That would be + // too limiting to macro authors. + ($str:expr) => {{ + const S: &str = concat!($str, "\0"); + const C: &$crate::str::CStr = match $crate::str::CStr::from_bytes_with_nul(S.as_bytes()) { + Ok(v) => v, + Err(_) => panic!("string contains interior NUL"), + }; + C + }}; +} + +#[cfg(CONFIG_RUST_STR_KUNIT_TEST)] +#[kunit_tests(rust_kernel_str)] +mod tests { + use super::*; + + impl From for Error { + #[inline] + fn from(_: core::ffi::FromBytesWithNulError) -> Error { + EINVAL + } + } + + macro_rules! format { + ($($f:tt)*) => ({ + CString::try_from_fmt(fmt!($($f)*))?.to_str()? + }) + } + + const ALL_ASCII_CHARS: &str = + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ + \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \ + !\"#$%&'()*+,-./0123456789:;<=>?@\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\ + \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\ + \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\ + \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\ + \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\ + \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\ + \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\ + \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\ + \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff"; + + #[test] + fn test_cstr_to_str() -> Result { + let cstr = c"\xf0\x9f\xa6\x80"; + let checked_str = cstr.to_str()?; + assert_eq!(checked_str, "🦀"); + Ok(()) + } + + #[test] + fn test_cstr_to_str_invalid_utf8() -> Result { + let cstr = c"\xc3\x28"; + assert!(cstr.to_str().is_err()); + Ok(()) + } + + #[test] + fn test_cstr_display() -> Result { + let hello_world = c"hello, world!"; + assert_eq!(format!("{hello_world}"), "hello, world!"); + let non_printables = c"\x01\x09\x0a"; + assert_eq!(format!("{non_printables}"), "\\x01\\x09\\x0a"); + let non_ascii = c"d\xe9j\xe0 vu"; + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); + let good_bytes = c"\xf0\x9f\xa6\x80"; + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); + Ok(()) + } + + #[test] + fn test_cstr_display_all_bytes() -> Result { + let mut bytes: [u8; 256] = [0; 256]; + // fill `bytes` with [1..=255] + [0] + for i in u8::MIN..=u8::MAX { + bytes[i as usize] = i.wrapping_add(1); + } + let cstr = CStr::from_bytes_with_nul(&bytes)?; + assert_eq!(format!("{cstr}"), ALL_ASCII_CHARS); + Ok(()) + } + + #[test] + fn test_cstr_debug() -> Result { + let hello_world = c"hello, world!"; + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); + let non_printables = c"\x01\x09\x0a"; + assert_eq!(format!("{non_printables:?}"), "\"\\x01\\t\\n\""); + let non_ascii = c"d\xe9j\xe0 vu"; + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); + Ok(()) + } + + #[test] + fn test_bstr_display() -> Result { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{hello_world}"), "hello, world!"); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{escapes}"), "_\\t_\\n_\\r_\\_'_\"_"); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{others}"), "\\x01"); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); + Ok(()) + } + + #[test] + fn test_bstr_debug() -> Result { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{escapes:?}"), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{others:?}"), "\"\\x01\""); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\""); + Ok(()) + } +} + +/// Allows formatting of [`fmt::Arguments`] into a raw buffer. +/// +/// It does not fail if callers write past the end of the buffer so that they can calculate the +/// size required to fit everything. +/// +/// # Invariants +/// +/// The memory region between `pos` (inclusive) and `end` (exclusive) is valid for writes if `pos` +/// is less than `end`. +pub struct RawFormatter { + // Use `usize` to use `saturating_*` functions. + beg: usize, + pos: usize, + end: usize, +} + +impl RawFormatter { + /// Creates a new instance of [`RawFormatter`] with an empty buffer. + fn new() -> Self { + // INVARIANT: The buffer is empty, so the region that needs to be writable is empty. + Self { + beg: 0, + pos: 0, + end: 0, + } + } + + /// Creates a new instance of [`RawFormatter`] with the given buffer pointers. + /// + /// # Safety + /// + /// If `pos` is less than `end`, then the region between `pos` (inclusive) and `end` + /// (exclusive) must be valid for writes for the lifetime of the returned [`RawFormatter`]. + pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self { + // INVARIANT: The safety requirements guarantee the type invariants. + Self { + beg: pos as usize, + pos: pos as usize, + end: end as usize, + } + } + + /// Creates a new instance of [`RawFormatter`] with the given buffer. + /// + /// # Safety + /// + /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes + /// for the lifetime of the returned [`RawFormatter`]. + pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self { + let pos = buf as usize; + // INVARIANT: We ensure that `end` is never less than `buf`, and the safety requirements + // guarantees that the memory region is valid for writes. + Self { + pos, + beg: pos, + end: pos.saturating_add(len), + } + } + + /// Returns the current insert position. + /// + /// N.B. It may point to invalid memory. + pub(crate) fn pos(&self) -> *mut u8 { + self.pos as *mut u8 + } + + /// Returns the number of bytes written to the formatter. + pub fn bytes_written(&self) -> usize { + self.pos - self.beg + } +} + +impl fmt::Write for RawFormatter { + fn write_str(&mut self, s: &str) -> fmt::Result { + // `pos` value after writing `len` bytes. This does not have to be bounded by `end`, but we + // don't want it to wrap around to 0. + let pos_new = self.pos.saturating_add(s.len()); + + // Amount that we can copy. `saturating_sub` ensures we get 0 if `pos` goes past `end`. + let len_to_copy = core::cmp::min(pos_new, self.end).saturating_sub(self.pos); + + if len_to_copy > 0 { + // SAFETY: If `len_to_copy` is non-zero, then we know `pos` has not gone past `end` + // yet, so it is valid for write per the type invariants. + unsafe { + core::ptr::copy_nonoverlapping( + s.as_bytes().as_ptr(), + self.pos as *mut u8, + len_to_copy, + ) + }; + } + + self.pos = pos_new; + Ok(()) + } +} + +/// Allows formatting of [`fmt::Arguments`] into a raw buffer. +/// +/// Fails if callers attempt to write more than will fit in the buffer. +pub struct Formatter<'a>(RawFormatter, PhantomData<&'a mut ()>); + +impl Formatter<'_> { + /// Creates a new instance of [`Formatter`] with the given buffer. + /// + /// # Safety + /// + /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes + /// for the lifetime of the returned [`Formatter`]. + pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self { + // SAFETY: The safety requirements of this function satisfy those of the callee. + Self(unsafe { RawFormatter::from_buffer(buf, len) }, PhantomData) + } + + /// Create a new [`Self`] instance. + pub fn new(buffer: &mut [u8]) -> Self { + // SAFETY: `buffer` is valid for writes for the entire length for + // the lifetime of `Self`. + unsafe { Formatter::from_buffer(buffer.as_mut_ptr(), buffer.len()) } + } +} + +impl Deref for Formatter<'_> { + type Target = RawFormatter; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Write for Formatter<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.0.write_str(s)?; + + // Fail the request if we go past the end of the buffer. + if self.0.pos > self.0.end { + Err(fmt::Error) + } else { + Ok(()) + } + } +} + +/// A mutable reference to a byte buffer where a string can be written into. +/// +/// The buffer will be automatically null terminated after the last written character. +/// +/// # Invariants +/// +/// * The first byte of `buffer` is always zero. +/// * The length of `buffer` is at least 1. +pub struct NullTerminatedFormatter<'a> { + buffer: &'a mut [u8], +} + +impl<'a> NullTerminatedFormatter<'a> { + /// Create a new [`Self`] instance. + pub fn new(buffer: &'a mut [u8]) -> Option> { + *(buffer.first_mut()?) = 0; + + // INVARIANT: + // - We wrote zero to the first byte above. + // - If buffer was not at least length 1, `buffer.first_mut()` would return None. + Some(Self { buffer }) + } +} + +impl Write for NullTerminatedFormatter<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + let len = bytes.len(); + + // We want space for a zero. By type invariant, buffer length is always at least 1, so no + // underflow. + if len > self.buffer.len() - 1 { + return Err(fmt::Error); + } + + let buffer = core::mem::take(&mut self.buffer); + // We break the zero start invariant for a short while. + buffer[..len].copy_from_slice(bytes); + // INVARIANT: We checked above that buffer will have size at least 1 after this assignment. + self.buffer = &mut buffer[len..]; + + // INVARIANT: We write zero to the first byte of the buffer. + self.buffer[0] = 0; + + Ok(()) + } +} + +/// # Safety +/// +/// - `string` must point to a null terminated string that is valid for read. +unsafe fn kstrtobool_raw(string: *const u8) -> Result { + let mut result: bool = false; + + // SAFETY: + // - By function safety requirement, `string` is a valid null-terminated string. + // - `result` is a valid `bool` that we own. + to_result(unsafe { bindings::kstrtobool(string, &mut result) })?; + Ok(result) +} + +/// Convert common user inputs into boolean values using the kernel's `kstrtobool` function. +/// +/// This routine returns `Ok(bool)` if the first character is one of 'YyTt1NnFf0', or +/// \[oO\]\[NnFf\] for "on" and "off". Otherwise it will return `Err(EINVAL)`. +/// +/// # Examples +/// +/// ``` +/// # use kernel::str::kstrtobool; +/// +/// // Lowercase +/// assert_eq!(kstrtobool(c"true"), Ok(true)); +/// assert_eq!(kstrtobool(c"tr"), Ok(true)); +/// assert_eq!(kstrtobool(c"t"), Ok(true)); +/// assert_eq!(kstrtobool(c"twrong"), Ok(true)); +/// assert_eq!(kstrtobool(c"false"), Ok(false)); +/// assert_eq!(kstrtobool(c"f"), Ok(false)); +/// assert_eq!(kstrtobool(c"yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"no"), Ok(false)); +/// assert_eq!(kstrtobool(c"on"), Ok(true)); +/// assert_eq!(kstrtobool(c"off"), Ok(false)); +/// +/// // Camel case +/// assert_eq!(kstrtobool(c"True"), Ok(true)); +/// assert_eq!(kstrtobool(c"False"), Ok(false)); +/// assert_eq!(kstrtobool(c"Yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"No"), Ok(false)); +/// assert_eq!(kstrtobool(c"On"), Ok(true)); +/// assert_eq!(kstrtobool(c"Off"), Ok(false)); +/// +/// // All caps +/// assert_eq!(kstrtobool(c"TRUE"), Ok(true)); +/// assert_eq!(kstrtobool(c"FALSE"), Ok(false)); +/// assert_eq!(kstrtobool(c"YES"), Ok(true)); +/// assert_eq!(kstrtobool(c"NO"), Ok(false)); +/// assert_eq!(kstrtobool(c"ON"), Ok(true)); +/// assert_eq!(kstrtobool(c"OFF"), Ok(false)); +/// +/// // Numeric +/// assert_eq!(kstrtobool(c"1"), Ok(true)); +/// assert_eq!(kstrtobool(c"0"), Ok(false)); +/// +/// // Invalid input +/// assert_eq!(kstrtobool(c"invalid"), Err(EINVAL)); +/// assert_eq!(kstrtobool(c"2"), Err(EINVAL)); +/// ``` +pub fn kstrtobool(string: &CStr) -> Result { + // SAFETY: + // - The pointer returned by `CStr::as_char_ptr` is guaranteed to be + // null terminated. + // - `string` is live and thus the string is valid for read. + unsafe { kstrtobool_raw(string.as_char_ptr()) } +} + +/// Convert `&[u8]` to `bool` by deferring to [`kernel::str::kstrtobool`]. +/// +/// Only considers at most the first two bytes of `bytes`. +pub fn kstrtobool_bytes(bytes: &[u8]) -> Result { + // `ktostrbool` only considers the first two bytes of the input. + let stack_string = [*bytes.first().unwrap_or(&0), *bytes.get(1).unwrap_or(&0), 0]; + // SAFETY: `stack_string` is null terminated and it is live on the stack so + // it is valid for read. + unsafe { kstrtobool_raw(stack_string.as_ptr()) } +} + +/// An owned string that is guaranteed to have exactly one `NUL` byte, which is at the end. +/// +/// Used for interoperability with kernel APIs that take C strings. +/// +/// # Invariants +/// +/// The string is always `NUL`-terminated and contains no other `NUL` bytes. +/// +/// # Examples +/// +/// ``` +/// use kernel::{str::CString, prelude::fmt}; +/// +/// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20))?; +/// assert_eq!(s.to_bytes_with_nul(), "abc1020\0".as_bytes()); +/// +/// let tmp = "testing"; +/// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123))?; +/// assert_eq!(s.to_bytes_with_nul(), "testing123\0".as_bytes()); +/// +/// // This fails because it has an embedded `NUL` byte. +/// let s = CString::try_from_fmt(fmt!("a\0b{}", 123)); +/// assert_eq!(s.is_ok(), false); +/// # Ok::<(), kernel::error::Error>(()) +/// ``` +pub struct CString { + buf: KVec, +} + +impl CString { + /// Creates an instance of [`CString`] from the given formatted arguments. + pub fn try_from_fmt(args: fmt::Arguments<'_>) -> Result { + // Calculate the size needed (formatted string plus `NUL` terminator). + let mut f = RawFormatter::new(); + f.write_fmt(args)?; + f.write_str("\0")?; + let size = f.bytes_written(); + + // Allocate a vector with the required number of bytes, and write to it. + let mut buf = KVec::with_capacity(size, GFP_KERNEL)?; + // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. + let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; + f.write_fmt(args)?; + f.write_str("\0")?; + + // SAFETY: The number of bytes that can be written to `f` is bounded by `size`, which is + // `buf`'s capacity. The `Formatter` is created with `size` as its limit, and the `?` + // operators on `write_fmt` and `write_str` above ensure that if writing exceeds this + // limit, an error is returned early. The contents of the buffer have been initialised + // by writes to `f`. + unsafe { buf.inc_len(f.bytes_written()) }; + + // Check that there are no `NUL` bytes before the end. + // SAFETY: The buffer is valid for read because `f.bytes_written()` is bounded by `size` + // (which the minimum buffer size) and is non-zero (we wrote at least the `NUL` terminator) + // so `f.bytes_written() - 1` doesn't underflow. + let ptr = unsafe { bindings::memchr(buf.as_ptr().cast(), 0, f.bytes_written() - 1) }; + if !ptr.is_null() { + return Err(EINVAL); + } + + // INVARIANT: We wrote the `NUL` terminator and checked above that no other `NUL` bytes + // exist in the buffer. + Ok(Self { buf }) + } +} + +impl Deref for CString { + type Target = CStr; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the string is `NUL`-terminated and that no + // other `NUL` bytes exist. + unsafe { CStr::from_bytes_with_nul_unchecked(self.buf.as_slice()) } + } +} + +impl DerefMut for CString { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: A `CString` is always NUL-terminated and contains no other + // NUL bytes. + unsafe { CStr::from_bytes_with_nul_unchecked_mut(self.buf.as_mut_slice()) } + } +} + +impl<'a> TryFrom<&'a CStr> for CString { + type Error = AllocError; + + fn try_from(cstr: &'a CStr) -> Result { + let mut buf = KVec::new(); + + buf.extend_from_slice(cstr.to_bytes_with_nul(), GFP_KERNEL)?; + + // INVARIANT: The `CStr` and `CString` types have the same invariants for + // the string data, and we copied it over without changes. + Ok(CString { buf }) + } +} + +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} diff --git a/rust/kernel/str/parse_int.rs b/rust/kernel/str/parse_int.rs new file mode 100644 index 000000000..48eb4c202 --- /dev/null +++ b/rust/kernel/str/parse_int.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Integer parsing functions. +//! +//! Integer parsing functions for parsing signed and unsigned integers +//! potentially prefixed with `0x`, `0o`, or `0b`. + +use crate::prelude::*; +use crate::str::BStr; +use core::ops::Deref; + +// Make `FromStrRadix` a public type with a private name. This seals +// `ParseInt`, that is, prevents downstream users from implementing the +// trait. +mod private { + use crate::prelude::*; + use crate::str::BStr; + + /// Trait that allows parsing a [`&BStr`] to an integer with a radix. + pub trait FromStrRadix: Sized { + /// Parse `src` to [`Self`] using radix `radix`. + fn from_str_radix(src: &BStr, radix: u32) -> Result; + + /// Tries to convert `value` into [`Self`] and negates the resulting value. + fn from_u64_negated(value: u64) -> Result; + } +} + +/// Extract the radix from an integer literal optionally prefixed with +/// one of `0x`, `0X`, `0o`, `0O`, `0b`, `0B`, `0`. +fn strip_radix(src: &BStr) -> (u32, &BStr) { + match src.deref() { + [b'0', b'x' | b'X', rest @ ..] => (16, rest.as_ref()), + [b'0', b'o' | b'O', rest @ ..] => (8, rest.as_ref()), + [b'0', b'b' | b'B', rest @ ..] => (2, rest.as_ref()), + // NOTE: We are including the leading zero to be able to parse + // literal `0` here. If we removed it as a radix prefix, we would + // not be able to parse `0`. + [b'0', ..] => (8, src), + _ => (10, src), + } +} + +/// Trait for parsing string representations of integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtoul +/// +/// # Examples +/// +/// ``` +/// # use kernel::str::parse_int::ParseInt; +/// # use kernel::b_str; +/// +/// assert_eq!(Ok(0u8), u8::from_str(b_str!("0"))); +/// +/// assert_eq!(Ok(0xa2u8), u8::from_str(b_str!("0xa2"))); +/// assert_eq!(Ok(-0xa2i32), i32::from_str(b_str!("-0xa2"))); +/// +/// assert_eq!(Ok(-0o57i8), i8::from_str(b_str!("-0o57"))); +/// assert_eq!(Ok(0o57i8), i8::from_str(b_str!("057"))); +/// +/// assert_eq!(Ok(0b1001i16), i16::from_str(b_str!("0b1001"))); +/// assert_eq!(Ok(-0b1001i16), i16::from_str(b_str!("-0b1001"))); +/// +/// assert_eq!(Ok(127i8), i8::from_str(b_str!("127"))); +/// assert!(i8::from_str(b_str!("128")).is_err()); +/// assert_eq!(Ok(-128i8), i8::from_str(b_str!("-128"))); +/// assert!(i8::from_str(b_str!("-129")).is_err()); +/// assert_eq!(Ok(255u8), u8::from_str(b_str!("255"))); +/// assert!(u8::from_str(b_str!("256")).is_err()); +/// ``` +pub trait ParseInt: private::FromStrRadix + TryFrom { + /// Parse a string according to the description in [`Self`]. + fn from_str(src: &BStr) -> Result { + match src.deref() { + [b'-', rest @ ..] => { + let (radix, digits) = strip_radix(rest.as_ref()); + // 2's complement values range from -2^(b-1) to 2^(b-1)-1. + // So if we want to parse negative numbers as positive and + // later multiply by -1, we have to parse into a larger + // integer. We choose `u64` as sufficiently large. + // + // NOTE: 128 bit integers are not available on all + // platforms, hence the choice of 64 bits. + let val = + u64::from_str_radix(core::str::from_utf8(digits).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL)?; + Self::from_u64_negated(val) + } + _ => { + let (radix, digits) = strip_radix(src); + Self::from_str_radix(digits, radix).map_err(|_| EINVAL) + } + } + } +} + +macro_rules! impl_parse_int { + ($($ty:ty),*) => { + $( + impl private::FromStrRadix for $ty { + fn from_str_radix(src: &BStr, radix: u32) -> Result { + <$ty>::from_str_radix(core::str::from_utf8(src).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL) + } + + fn from_u64_negated(value: u64) -> Result { + const ABS_MIN: u64 = { + #[allow(unused_comparisons)] + if <$ty>::MIN < 0 { + 1u64 << (<$ty>::BITS - 1) + } else { + 0 + } + }; + + if value > ABS_MIN { + return Err(EINVAL); + } + + if value == ABS_MIN { + return Ok(<$ty>::MIN); + } + + // SAFETY: The above checks guarantee that `value` fits into `Self`: + // - if `Self` is unsigned, then `ABS_MIN == 0` and thus we have returned above + // (either `EINVAL` or `MIN`). + // - if `Self` is signed, then we have that `0 <= value < ABS_MIN`. And since + // `ABS_MIN - 1` fits into `Self` by construction, `value` also does. + let value: Self = unsafe { value.try_into().unwrap_unchecked() }; + + Ok((!value).wrapping_add(1)) + } + } + + impl ParseInt for $ty {} + )* + }; +} + +impl_parse_int![i8, u8, i16, u16, i32, u32, i64, u64, isize, usize]; diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs new file mode 100644 index 000000000..3e55de3b1 --- /dev/null +++ b/rust/kernel/sync.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Synchronisation primitives. +//! +//! This module contains the kernel APIs related to synchronisation that have been ported or +//! wrapped for usage by Rust code in the kernel. + +use crate::prelude::*; +use crate::types::Opaque; +use pin_init; + +mod arc; +pub mod aref; +pub mod atomic; +pub mod barrier; +pub mod completion; +mod condvar; +pub mod lock; +mod locked_by; +pub mod poll; +pub mod rcu; +mod refcount; +mod set_once; +pub mod srcu; + +pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use completion::Completion; +pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; +pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; +pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; +pub use locked_by::LockedBy; +pub use refcount::Refcount; +pub use set_once::SetOnce; +pub use srcu::Srcu; + +/// Represents a lockdep class. +/// +/// Wraps the kernel's `struct lock_class_key`. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct LockClassKey { + #[pin] + inner: Opaque, +} + +// SAFETY: Unregistering a lock class key from a different thread than where it was registered is +// allowed. +unsafe impl Send for LockClassKey {} + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + +impl LockClassKey { + /// Initializes a statically allocated lock class key. + /// + /// This is usually used indirectly through the [`static_lock_class!`] macro. See its + /// documentation for more information. + /// + /// # Safety + /// + /// * Before using the returned value, it must be pinned in a static memory location. + /// * The destructor must never run on the returned `LockClassKey`. + pub const unsafe fn new_static() -> Self { + LockClassKey { + inner: Opaque::uninit(), + } + } + + /// Initializes a dynamically allocated lock class key. + /// + /// In the common case of using a statically allocated lock class key, the + /// [`static_lock_class!`] macro should be used instead. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::KBox; + /// use kernel::types::ForeignOwnable; + /// use kernel::sync::{LockClassKey, SpinLock}; + /// use pin_init::stack_pin_init; + /// + /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; + /// let key_ptr = key.into_foreign(); + /// + /// { + /// stack_pin_init!(let num: SpinLock = SpinLock::new( + /// 0, + /// c"my_spinlock", + /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose + /// // `from_foreign()` has not yet been called. + /// unsafe { > as ForeignOwnable>::borrow(key_ptr) } + /// )); + /// } + /// + /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous + /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. + /// unsafe { drop(> as ForeignOwnable>::from_foreign(key_ptr)) }; + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_dynamic() -> impl PinInit { + pin_init!(Self { + // SAFETY: lockdep_register_key expects an uninitialized block of memory + inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) + }) + } + + /// Returns a raw pointer to the inner C struct. + /// + /// It is up to the caller to use the raw pointer correctly. + pub fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for LockClassKey { + fn drop(self: Pin<&mut Self>) { + // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address + // hasn't changed. Thus, it's safe to pass it to unregister. + unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } + } +} + +/// Defines a new static lock class and returns a pointer to it. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{static_lock_class, Arc, SpinLock}; +/// +/// fn new_locked_int() -> Result>> { +/// Arc::pin_init(SpinLock::new( +/// 42, +/// c"new_locked_int", +/// static_lock_class!(), +/// ), GFP_KERNEL) +/// } +/// ``` +#[macro_export] +macro_rules! static_lock_class { + () => {{ + static CLASS: $crate::sync::LockClassKey = + // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop + // never runs on a static global. + unsafe { $crate::sync::LockClassKey::new_static() }; + $crate::prelude::Pin::static_ref(&CLASS) + }}; +} +pub use static_lock_class; + +/// Returns the given string, if one is provided, otherwise generates one based on the source code +/// location. +#[doc(hidden)] +#[macro_export] +macro_rules! optional_name { + () => { + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) + }; + ($name:literal) => { + $crate::c_str!($name) + }; +} diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs new file mode 100644 index 000000000..8ae0fe6f1 --- /dev/null +++ b/rust/kernel/sync/arc.rs @@ -0,0 +1,918 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A reference-counted pointer. +//! +//! This module implements a way for users to create reference-counted objects and pointers to +//! them. Such a pointer automatically increments and decrements the count, and drops the +//! underlying object when it reaches zero. It is also safe to use concurrently from multiple +//! threads. +//! +//! It is different from the standard library's [`Arc`] in a few ways: +//! 1. It is backed by the kernel's [`Refcount`] type. +//! 2. It does not support weak references, which allows it to be half the size. +//! 3. It saturates the reference count instead of aborting when it goes over a threshold. +//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. +//! 5. The object in [`Arc`] is pinned implicitly. +//! +//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html + +use crate::{ + alloc::{AllocError, Flags, KBox}, + ffi::c_void, + fmt, + init::InPlaceInit, + sync::Refcount, + try_init, + types::ForeignOwnable, +}; +use core::{ + alloc::Layout, + borrow::{Borrow, BorrowMut}, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, + ops::{Deref, DerefMut}, + pin::Pin, + ptr::NonNull, +}; +use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit}; + +mod std_vendor; + +/// A reference-counted pointer to an instance of `T`. +/// +/// The reference count is incremented when new instances of [`Arc`] are created, and decremented +/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. +/// +/// # Invariants +/// +/// The reference count on an instance of [`Arc`] is always non-zero. +/// The object pointed to by [`Arc`] is always pinned. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// // Create a refcounted instance of `Example`. +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// +/// // Get a new pointer to `obj` and increment the refcount. +/// let cloned = obj.clone(); +/// +/// // Assert that both `obj` and `cloned` point to the same underlying object. +/// assert!(core::ptr::eq(&*obj, &*cloned)); +/// +/// // Destroy `obj` and decrement its refcount. +/// drop(obj); +/// +/// // Check that the values are still accessible through `cloned`. +/// assert_eq!(cloned.a, 10); +/// assert_eq!(cloned.b, 20); +/// +/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Using `Arc` as the type of `self`: +/// +/// ``` +/// use kernel::sync::Arc; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// impl Example { +/// fn take_over(self: Arc) { +/// // ... +/// } +/// +/// fn use_reference(self: &Arc) { +/// // ... +/// } +/// } +/// +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// obj.use_reference(); +/// obj.take_over(); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Coercion from `Arc` to `Arc`: +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// trait MyTrait { +/// // Trait has a function whose `self` type is `Arc`. +/// fn example1(self: Arc) {} +/// +/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`. +/// fn example2(self: ArcBorrow<'_, Self>) {} +/// } +/// +/// struct Example; +/// impl MyTrait for Example {} +/// +/// // `obj` has type `Arc`. +/// let obj: Arc = Arc::new(Example, GFP_KERNEL)?; +/// +/// // `coerced` has type `Arc`. +/// let coerced: Arc = obj; +/// # Ok::<(), Error>(()) +/// ``` +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct Arc { + ptr: NonNull>, + // NB: this informs dropck that objects of type `ArcInner` may be used in ` as + // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in + // ` as Drop>::drop` and the distinction between `T` and `ArcInner` is not presently + // meaningful with respect to dropck - but this may change in the future so this is left here + // out of an abundance of caution. + // + // See + // for more detail on the semantics of dropck in the presence of `PhantomData`. + _p: PhantomData>, +} + +#[pin_data] +#[repr(C)] +struct ArcInner { + refcount: Refcount, + data: T, +} + +impl ArcInner { + /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must + /// not yet have been destroyed. + unsafe fn container_of(ptr: *const T) -> NonNull> { + let refcount_layout = Layout::new::(); + // SAFETY: The caller guarantees that the pointer is valid. + let val_layout = Layout::for_value(unsafe { &*ptr }); + // SAFETY: We're computing the layout of a real struct that existed when compiling this + // binary, so its layout is not so large that it can trigger arithmetic overflow. + let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; + + // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and + // `ArcInner` is the same since `ArcInner` is a struct with `T` as its last field. + // + // This is documented at: + // . + let ptr = ptr as *const ArcInner; + + // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the + // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is + // still valid. + let ptr = unsafe { ptr.byte_sub(val_offset) }; + + // SAFETY: The pointer can't be null since you can't have an `ArcInner` value at the null + // address. + unsafe { NonNull::new_unchecked(ptr.cast_mut()) } + } +} + +// SAFETY: It is safe to send `Arc` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has an `Arc` may ultimately access `T` using a +// mutable reference when the reference count reaches zero and `T` is dropped. +unsafe impl Send for Arc {} + +// SAFETY: It is safe to send `&Arc` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, +// it needs `T` to be `Send` because any thread that has a `&Arc` may clone it and get an +// `Arc` on that thread, so the thread may ultimately access `T` using a mutable reference when +// the reference count reaches zero and `T` is dropped. +unsafe impl Sync for Arc {} + +impl InPlaceInit for Arc { + type PinnedSelf = Self; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_pin_init(init, flags).map(|u| u.into()) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_init(init, flags).map(|u| u.into()) + } +} + +impl Arc { + /// Constructs a new reference counted instance of `T`. + pub fn new(contents: T, flags: Flags) -> Result { + // INVARIANT: The refcount is initialised to a non-zero value. + let value = ArcInner { + refcount: Refcount::new(1), + data: contents, + }; + + let inner = KBox::new(value, flags)?; + let inner = KBox::leak(inner).into(); + + // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new + // `Arc` object. + Ok(unsafe { Self::from_inner(inner) }) + } + + /// The offset that the value is stored at. + pub const DATA_OFFSET: usize = core::mem::offset_of!(ArcInner, data); +} + +impl Arc { + /// Constructs a new [`Arc`] from an existing [`ArcInner`]. + /// + /// # Safety + /// + /// The caller must ensure that `inner` points to a valid location and has a non-zero reference + /// count, one of which will be owned by the new [`Arc`] instance. + unsafe fn from_inner(inner: NonNull>) -> Self { + // INVARIANT: By the safety requirements, the invariants hold. + Arc { + ptr: inner, + _p: PhantomData, + } + } + + /// Convert the [`Arc`] into a raw pointer. + /// + /// The raw pointer has ownership of the refcount that this [`Arc`] object owned. + pub fn into_raw(self) -> *const T { + let ptr = self.ptr.as_ptr(); + core::mem::forget(self); + // SAFETY: The pointer is valid. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Return a raw pointer to the data in this [`Arc`]. + pub fn as_ptr(this: &Self) -> *const T { + let ptr = this.ptr.as_ptr(); + + // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`, + // field projection to `data`is within bounds of the allocation. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it + /// must not be called more than once for each previous call to [`Arc::into_raw`]. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the + // reference count held then will be owned by the new `Arc` object. + unsafe { Self::from_inner(ptr) } + } + + /// Returns an [`ArcBorrow`] from the given [`Arc`]. + /// + /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method + /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. + #[inline] + pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { + // SAFETY: The constraint that the lifetime of the shared reference must outlive that of + // the returned `ArcBorrow` ensures that the object remains alive and that no mutable + // reference can be created. + unsafe { ArcBorrow::new(self.ptr) } + } + + /// Compare whether two [`Arc`] pointers reference the same underlying object. + pub fn ptr_eq(this: &Self, other: &Self) -> bool { + core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) + } + + /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. + /// + /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that + /// this method will never call the destructor of the value. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let unique_arc = Arc::into_unique_or_drop(arc); + /// + /// // The above conversion should succeed since refcount of `arc` is 1. + /// assert!(unique_arc.is_some()); + /// + /// assert_eq!(*(unique_arc.unwrap()), 42); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let another = arc.clone(); + /// + /// let unique_arc = Arc::into_unique_or_drop(arc); + /// + /// // The above conversion should fail since refcount of `arc` is >1. + /// assert!(unique_arc.is_none()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn into_unique_or_drop(this: Self) -> Option>> { + // We will manually manage the refcount in this method, so we disable the destructor. + let this = ManuallyDrop::new(this); + // SAFETY: We own a refcount, so the pointer is still valid. + let refcount = unsafe { &this.ptr.as_ref().refcount }; + + // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will + // return without further touching the `Arc`. If the refcount reaches zero, then there are + // no other `Arc`s, and we can create a `UniqueArc`. + if refcount.dec_and_test() { + refcount.set(1); + + // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We + // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin + // their values. + Some(Pin::from(UniqueArc { + inner: ManuallyDrop::into_inner(this), + })) + } else { + None + } + } +} + +// SAFETY: The pointer returned by `into_foreign` was originally allocated as an +// `KBox>`, so that type is what determines the alignment. +unsafe impl ForeignOwnable for Arc { + const FOREIGN_ALIGN: usize = > as ForeignOwnable>::FOREIGN_ALIGN; + + type Borrowed<'a> = ArcBorrow<'a, T>; + type BorrowedMut<'a> = Self::Borrowed<'a>; + + fn into_foreign(self) -> *mut c_void { + ManuallyDrop::new(self).ptr.as_ptr().cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::>()) }; + + // SAFETY: By the safety requirement of this function, we know that `ptr` came from + // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and + // holds a reference count increment that is transferrable to us. + unsafe { Self::from_inner(inner) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::>()) }; + + // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive + // for the lifetime of the returned value. + unsafe { ArcBorrow::new(inner) } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { ::borrow(ptr) } + } +} + +impl Deref for Arc { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to dereference it. + unsafe { &self.ptr.as_ref().data } + } +} + +impl AsRef for Arc { + fn as_ref(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::Arc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Shared instance. +/// let arc = Arc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc.clone()); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for Arc { + fn borrow(&self) -> &T { + self.deref() + } +} + +impl Clone for Arc { + fn clone(&self) -> Self { + // INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero. + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to increment the refcount. + unsafe { self.ptr.as_ref() }.refcount.inc(); + + // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. + unsafe { Self::from_inner(self.ptr) } + } +} + +impl Drop for Arc { + fn drop(&mut self) { + // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and + // this instance is being dropped, so the broken invariant is not observable. + // SAFETY: By the type invariant, there is necessarily a reference to the object. + let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test(); + if is_zero { + // The count reached zero, we must free the memory. + // + // SAFETY: The pointer was initialised from the result of `KBox::leak`. + unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; + } + } +} + +impl From> for Arc { + fn from(item: UniqueArc) -> Self { + item.inner + } +} + +impl From>> for Arc { + fn from(item: Pin>) -> Self { + // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. + unsafe { Pin::into_inner_unchecked(item).inner } + } +} + +/// A borrowed reference to an [`Arc`] instance. +/// +/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler +/// to use just `&T`, which we can trivially get from an [`Arc`] instance. +/// +/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow` +/// over `&Arc` because the latter results in a double-indirection: a pointer (shared reference) +/// to a pointer ([`Arc`]) to the object (`T`). An [`ArcBorrow`] eliminates this double +/// indirection while still allowing one to increment the refcount and getting an [`Arc`] when/if +/// needed. +/// +/// # Invariants +/// +/// There are no mutable references to the underlying [`Arc`], and it remains valid for the +/// lifetime of the [`ArcBorrow`] instance. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// struct Example; +/// +/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc { +/// e.into() +/// } +/// +/// let obj = Arc::new(Example, GFP_KERNEL)?; +/// let cloned = do_something(obj.as_arc_borrow()); +/// +/// // Assert that both `obj` and `cloned` point to the same underlying object. +/// assert!(core::ptr::eq(&*obj, &*cloned)); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Using `ArcBorrow` as the type of `self`: +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// impl Example { +/// fn use_reference(self: ArcBorrow<'_, Self>) { +/// // ... +/// } +/// } +/// +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// obj.as_arc_borrow().use_reference(); +/// # Ok::<(), Error>(()) +/// ``` +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct ArcBorrow<'a, T: ?Sized + 'a> { + inner: NonNull>, + _p: PhantomData<&'a ()>, +} + +impl Clone for ArcBorrow<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for ArcBorrow<'_, T> {} + +impl ArcBorrow<'_, T> { + /// Creates a new [`ArcBorrow`] instance. + /// + /// # Safety + /// + /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance: + /// 1. That `inner` remains valid; + /// 2. That no mutable references to `inner` are created. + unsafe fn new(inner: NonNull>) -> Self { + // INVARIANT: The safety requirements guarantee the invariants. + Self { + inner, + _p: PhantomData, + } + } + + /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with + /// [`Arc::into_raw`] or [`Arc::as_ptr`]. + /// + /// # Safety + /// + /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`]. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must + /// not hit zero. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a + /// [`UniqueArc`] reference to this value. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: The caller promises that the value remains valid since the reference count must + // not hit zero, and no mutable reference will be created since that would involve a + // `UniqueArc`. + unsafe { Self::new(ptr) } + } +} + +impl From> for Arc { + fn from(b: ArcBorrow<'_, T>) -> Self { + // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop` + // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the + // increment. + ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) }) + .deref() + .clone() + } +} + +impl Deref for ArcBorrow<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant, the underlying object is still alive with no mutable + // references to it, so it is safe to create a shared reference. + unsafe { &self.inner.as_ref().data } + } +} + +/// A refcounted object that is known to have a refcount of 1. +/// +/// It is mutable and can be converted to an [`Arc`] so that it can be shared. +/// +/// # Invariants +/// +/// `inner` always has a reference count of 1. +/// +/// # Examples +/// +/// In the following example, we make changes to the inner object before turning it into an +/// `Arc` object (after which point, it cannot be mutated directly). Note that `x.into()` +/// cannot fail. +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// x.a += 1; +/// x.b += 1; +/// Ok(x.into()) +/// } +/// +/// # test().unwrap(); +/// ``` +/// +/// In the following example we first allocate memory for a refcounted `Example` but we don't +/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], +/// followed by a conversion to `Arc`. This is particularly useful when allocation happens +/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let x = UniqueArc::new_uninit(GFP_KERNEL)?; +/// Ok(x.write(Example { a: 10, b: 20 }).into()) +/// } +/// +/// # test().unwrap(); +/// ``` +/// +/// In the last example below, the caller gets a pinned instance of `Example` while converting to +/// `Arc`; this is useful in scenarios where one needs a pinned reference during +/// initialisation, for example, when initialising fields that are wrapped in locks. +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); +/// // We can modify `pinned` because it is `Unpin`. +/// pinned.as_mut().a += 1; +/// Ok(pinned.into()) +/// } +/// +/// # test().unwrap(); +/// ``` +pub struct UniqueArc { + inner: Arc, +} + +impl InPlaceInit for UniqueArc { + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + UniqueArc::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + UniqueArc::new_uninit(flags)?.write_init(init) + } +} + +impl InPlaceWrite for UniqueArc> { + type Initialized = UniqueArc; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} + +impl UniqueArc { + /// Tries to allocate a new [`UniqueArc`] instance. + pub fn new(value: T, flags: Flags) -> Result { + Ok(Self { + // INVARIANT: The newly-created object has a refcount of 1. + inner: Arc::new(value, flags)?, + }) + } + + /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. + pub fn new_uninit(flags: Flags) -> Result>, AllocError> { + // INVARIANT: The refcount is initialised to a non-zero value. + let inner = KBox::try_init::( + try_init!(ArcInner { + refcount: Refcount::new(1), + data <- pin_init::uninit::(), + }? AllocError), + flags, + )?; + Ok(UniqueArc { + // INVARIANT: The newly-created object has a refcount of 1. + // SAFETY: The pointer from the `KBox` is valid. + inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) }, + }) + } +} + +impl UniqueArc { + /// Return a raw pointer to the data in this [`UniqueArc`]. + #[inline] + pub fn as_ptr(this: &Self) -> *const T { + Arc::as_ptr(&this.inner) + } +} + +impl UniqueArc> { + /// Converts a `UniqueArc>` into a `UniqueArc` by writing a value into it. + pub fn write(mut self, value: T) -> UniqueArc { + self.deref_mut().write(value); + // SAFETY: We just wrote the value to be initialized. + unsafe { self.assume_init() } + } + + /// Unsafely assume that `self` is initialized. + /// + /// # Safety + /// + /// The caller guarantees that the value behind this pointer has been initialized. It is + /// *immediate* UB to call this when the value is not initialized. + pub unsafe fn assume_init(self) -> UniqueArc { + let inner = ManuallyDrop::new(self).inner.ptr; + UniqueArc { + // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be + // dropped). The types are compatible because `MaybeUninit` is compatible with `T`. + inner: unsafe { Arc::from_inner(inner.cast()) }, + } + } + + /// Initialize `self` using the given initializer. + #[inline] + pub fn init_with(mut self, init: impl Init) -> core::result::Result, E> { + // SAFETY: The supplied pointer is valid for initialization. + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }), + Err(err) => Err(err), + } + } + + /// Pin-initialize `self` using the given pin-initializer. + #[inline] + pub fn pin_init_with( + mut self, + init: impl PinInit, + ) -> core::result::Result>, E> { + // SAFETY: The supplied pointer is valid for initialization and we will later pin the value + // to ensure it does not move. + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }.into()), + Err(err) => Err(err), + } + } +} + +impl From> for Pin> { + fn from(obj: UniqueArc) -> Self { + // SAFETY: It is not possible to move/replace `T` inside a `Pin>` (unless `T` + // is `Unpin`), so it is ok to convert it to `Pin>`. + unsafe { Pin::new_unchecked(obj) } + } +} + +impl Deref for UniqueArc { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl DerefMut for UniqueArc { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so + // it is safe to dereference it. Additionally, we know there is only one reference when + // it's inside a `UniqueArc`, so it is safe to get a mutable reference. + unsafe { &mut self.inner.ptr.as_mut().data } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::UniqueArc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for UniqueArc { + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// # use kernel::sync::UniqueArc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut for UniqueArc { + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +impl fmt::Display for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Display for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Debug for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} + +impl fmt::Debug for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} diff --git a/rust/kernel/sync/arc/std_vendor.rs b/rust/kernel/sync/arc/std_vendor.rs new file mode 100644 index 000000000..11b3f4ecc --- /dev/null +++ b/rust/kernel/sync/arc/std_vendor.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . + +use crate::sync::{arc::ArcInner, Arc}; +use core::any::Any; + +impl Arc { + /// Attempt to downcast the `Arc` to a concrete type. + pub fn downcast(self) -> core::result::Result, Self> + where + T: Any + Send + Sync, + { + if (*self).is::() { + // SAFETY: We have just checked that the type is correct, so we can cast the pointer. + unsafe { + let ptr = self.ptr.cast::>(); + core::mem::forget(self); + Ok(Arc::from_inner(ptr)) + } + } else { + Err(self) + } + } +} diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs new file mode 100644 index 000000000..9983ee085 --- /dev/null +++ b/rust/kernel/sync/aref.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Internal reference counting support. +//! +//! Many C types already have their own reference counting mechanism (e.g. by storing a +//! `refcount_t`). This module provides support for directly using their internal reference count +//! from Rust; instead of making users have to use an additional Rust-reference count in the form of +//! [`Arc`]. +//! +//! The smart pointer [`ARef`] acts similarly to [`Arc`] in that it holds a refcount on the +//! underlying object, but this refcount is internal to the object. It essentially is a Rust +//! implementation of the `get_` and `put_` pattern used in C for reference counting. +//! +//! To make use of [`ARef`], `MyType` needs to implement [`AlwaysRefCounted`]. It is a trait +//! for accessing the internal reference count of an object of the `MyType` type. +//! +//! [`Arc`]: crate::sync::Arc +//! [`Arc`]: crate::sync::Arc + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + types::ForeignOwnable, // +}; + +/// Types that are _always_ reference counted. +/// +/// It allows such types to define their own custom ref increment and decrement functions. +/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference +/// [`ARef`]. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. For +/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted +/// instances of a type. +/// +/// # Safety +/// +/// Implementers must ensure that increments to the reference count keep the object alive in memory +/// at least until matching decrements are performed. +/// +/// Implementers must also ensure that all instances are reference-counted. (Otherwise they +/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object +/// alive.) +pub unsafe trait AlwaysRefCounted { + /// Increments the reference count on the object. + fn inc_ref(&self); + + /// Decrements the reference count on the object. + /// + /// Frees the object when the count reaches zero. + /// + /// # Safety + /// + /// Callers must ensure that there was a previous matching increment to the reference count, + /// and that the object is no longer used after its reference count is decremented (as it may + /// result in the object being freed), unless the caller owns another increment on the refcount + /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls + /// [`AlwaysRefCounted::dec_ref`] once). + unsafe fn dec_ref(obj: NonNull); +} + +/// An owned reference to an always-reference-counted object. +/// +/// The object's reference count is automatically decremented when an instance of [`ARef`] is +/// dropped. It is also automatically incremented when a new instance is created via +/// [`ARef::clone`]. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In +/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. +pub struct ARef { + ptr: NonNull, + _p: PhantomData, +} + +// SAFETY: It is safe to send `ARef` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has an `ARef` may ultimately access `T` using a +// mutable reference, for example, when the reference count reaches zero and `T` is dropped. +unsafe impl Send for ARef {} + +// SAFETY: It is safe to send `&ARef` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, +// it needs `T` to be `Send` because any thread that has a `&ARef` may clone it and get an +// `ARef` on that thread, so the thread may ultimately access `T` using a mutable reference, for +// example, when the reference count reaches zero and `T` is dropped. +unsafe impl Sync for ARef {} + +// Even if `T` is pinned, pointers to `T` can still move. +impl Unpin for ARef {} + +impl ARef { + /// Creates a new instance of [`ARef`]. + /// + /// It takes over an increment of the reference count on the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the reference count was incremented at least once, and that they + /// are properly relinquishing one increment. That is, if there is only one increment, callers + /// must not use the underlying object anymore -- it is only safe to do so via the newly + /// created [`ARef`]. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // increment on the refcount. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `ARef`, returning a raw pointer. + /// + /// This function does not change the refcount. After calling this function, the caller is + /// responsible for the refcount previously managed by the `ARef`. + /// + /// # Examples + /// + /// ``` + /// use core::ptr::NonNull; + /// use kernel::sync::aref::{ARef, AlwaysRefCounted}; + /// + /// struct Empty {} + /// + /// # // SAFETY: TODO. + /// unsafe impl AlwaysRefCounted for Empty { + /// fn inc_ref(&self) {} + /// unsafe fn dec_ref(_obj: NonNull) {} + /// } + /// + /// let mut data = Empty {}; + /// let ptr = NonNull::::new(&mut data).unwrap(); + /// # // SAFETY: TODO. + /// let data_ref: ARef = unsafe { ARef::from_raw(ptr) }; + /// let raw_ptr: NonNull = ARef::into_raw(data_ref); + /// + /// assert_eq!(ptr, raw_ptr); + /// ``` + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } +} + +impl Clone for ARef { + fn clone(&self) -> Self { + self.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(self.ptr) } + } +} + +impl Deref for ARef { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl From<&T> for ARef { + fn from(b: &T) -> Self { + b.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(NonNull::from(b)) } + } +} + +impl Drop for ARef { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to + // decrement. + unsafe { T::dec_ref(self.ptr) }; + } +} + +impl PartialEq> for ARef +where + T: AlwaysRefCounted + PartialEq, + U: AlwaysRefCounted, +{ + #[inline] + fn eq(&self, other: &ARef) -> bool { + T::eq(&**self, &**other) + } +} +impl Eq for ARef {} + +// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The +// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`. +unsafe impl ForeignOwnable for ARef { + const FOREIGN_ALIGN: usize = core::mem::align_of::(); + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a T + where + Self: 'a; + + #[inline] + fn into_foreign(self) -> *mut c_void { + ARef::into_raw(self).as_ptr().cast() + } + + #[inline] + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing + // the refcount, so we can transfer the ownership to the new `ARef`. + unsafe { ARef::from_raw(ptr) } + } + + #[inline] + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + #[inline] + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { ::borrow(ptr) } + } +} + +impl PartialEq<&'_ U> for ARef +where + T: AlwaysRefCounted + PartialEq, +{ + #[inline] + fn eq(&self, other: &&U) -> bool { + T::eq(&**self, other) + } +} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs new file mode 100644 index 000000000..9cd009d57 --- /dev/null +++ b/rust/kernel/sync/atomic.rs @@ -0,0 +1,850 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic primitives. +//! +//! These primitives have the same semantics as their C counterparts: and the precise definitions of +//! semantics can be found at [`LKMM`]. Note that Linux Kernel Memory (Consistency) Model is the +//! only model for Rust code in kernel, and Rust's own atomics should be avoided. +//! +//! # Data races +//! +//! [`LKMM`] atomics have different rules regarding data races: +//! +//! - A normal write from C side is treated as an atomic write if +//! CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=y. +//! - Mixed-size atomic accesses don't cause data races. +//! +//! [`LKMM`]: srctree/tools/memory-model/ + +mod internal; +pub mod ordering; +mod predefine; + +pub use internal::AtomicImpl; +pub use ordering::{Acquire, Full, Relaxed, Release}; + +pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps}; + +use crate::build_error; +use internal::AtomicRepr; +use ordering::OrderingType; + +/// A memory location which can be safely modified from multiple execution contexts. +/// +/// This has the same size, alignment and bit validity as the underlying type `T`. And it disables +/// niche optimization for the same reason as [`UnsafeCell`]. +/// +/// The atomic operations are implemented in a way that is fully compatible with the [Linux Kernel +/// Memory (Consistency) Model][LKMM], hence they should be modeled as the corresponding +/// [`LKMM`][LKMM] atomic primitives. With the help of [`Atomic::from_ptr()`] and +/// [`Atomic::as_ptr()`], this provides a way to interact with [C-side atomic operations] +/// (including those without the `atomic` prefix, e.g. `READ_ONCE()`, `WRITE_ONCE()`, +/// `smp_load_acquire()` and `smp_store_release()`). +/// +/// # Invariants +/// +/// `self.0` is a valid `T`. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +/// [LKMM]: srctree/tools/memory-model/ +/// [C-side atomic operations]: srctree/Documentation/atomic_t.txt +#[repr(transparent)] +pub struct Atomic(AtomicRepr); + +// SAFETY: `Atomic` is safe to transfer between execution contexts because of the safety +// requirement of `AtomicType`. +unsafe impl Send for Atomic {} + +// SAFETY: `Atomic` is safe to share among execution contexts because all accesses are atomic. +unsafe impl Sync for Atomic {} + +/// Types that support basic atomic operations. +/// +/// # Round-trip transmutability +/// +/// `T` is round-trip transmutable to `U` if and only if both of these properties hold: +/// +/// - Any valid bit pattern for `T` is also a valid bit pattern for `U`. +/// - Transmuting (e.g. using [`transmute()`]) a value of type `T` to `U` and then to `T` again +/// yields a value that is in all aspects equivalent to the original value. +/// +/// # Safety +/// +/// - [`Self`] must have the same size and alignment as [`Self::Repr`]. +/// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. +/// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is +/// automatically satisfied. The exception is pointer types that are even though marked as +/// `!Send` (e.g. raw pointers and [`NonNull`]) but requiring `unsafe` to do anything +/// meaningful on them. This is because transferring pointer values between execution contexts is +/// safe as long as the actual `unsafe` dereferencing is justified. +/// +/// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. +/// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic +/// variables over unit-only enums, see [Examples]. +/// +/// # Limitations +/// +/// Because C primitives are used to implement the atomic operations, and a C function requires a +/// valid object of a type to operate on (i.e. no `MaybeUninit<_>`), hence at the Rust <-> C +/// surface, only types with all the bits initialized can be passed. As a result, types like `(u8, +/// u16)` (padding bytes are uninitialized) are currently not supported. +/// +/// # Examples +/// +/// A unit-only enum that implements [`AtomicType`]: +/// +/// ``` +/// use kernel::sync::atomic::{AtomicType, Atomic, Relaxed}; +/// +/// #[derive(Clone, Copy, PartialEq, Eq)] +/// #[repr(i32)] +/// enum State { +/// Uninit = 0, +/// Working = 1, +/// Done = 2, +/// }; +/// +/// // SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip +/// // transmutable to `i32`. +/// unsafe impl AtomicType for State { +/// type Repr = i32; +/// } +/// +/// let s = Atomic::new(State::Uninit); +/// +/// assert_eq!(State::Uninit, s.load(Relaxed)); +/// ``` +/// [`transmute()`]: core::mem::transmute +/// [round-trip transmutable]: AtomicType#round-trip-transmutability +/// [Examples]: AtomicType#examples +/// [`NonNull`]: core::ptr::NonNull +pub unsafe trait AtomicType: Sized + Copy { + /// The backing atomic implementation type. + type Repr: AtomicImpl; +} + +/// Types that support atomic add operations. +/// +/// # Safety +/// +// TODO: Properly defines `wrapping_add` in the following comment. +/// `wrapping_add` any value of type `Self::Repr::Delta` obtained by [`Self::rhs_into_delta()`] to +/// any value of type `Self::Repr` obtained through transmuting a value of type `Self` to must +/// yield a value with a bit pattern also valid for `Self`. +pub unsafe trait AtomicAdd: AtomicType { + /// Converts `Rhs` into the `Delta` type of the atomic implementation. + fn rhs_into_delta(rhs: Rhs) -> ::Delta; +} + +#[inline(always)] +const fn into_repr(v: T) -> T::Repr { + // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to + // `T::Repr`, therefore the transmute operation is sound. + unsafe { core::mem::transmute_copy(&v) } +} + +/// # Safety +/// +/// `r` must be a valid bit pattern of `T`. +#[inline(always)] +const unsafe fn from_repr(r: T::Repr) -> T { + // SAFETY: Per the safety requirement of the function, the transmute operation is sound. + unsafe { core::mem::transmute_copy(&r) } +} + +impl Atomic { + /// Creates a new atomic `T`. + pub const fn new(v: T) -> Self { + // INVARIANT: Per the safety requirement of `AtomicType`, `into_repr(v)` is a valid `T`. + Self(AtomicRepr::new(into_repr(v))) + } + + /// Creates a reference to an atomic `T` from a pointer of `T`. + /// + /// This usually is used when communicating with C side or manipulating a C struct, see + /// examples below. + /// + /// # Safety + /// + /// - `ptr` is aligned to `align_of::()`. + /// - `ptr` is valid for reads and writes for `'a`. + /// - For the duration of `'a`, other accesses to `*ptr` must not cause data races (defined + /// by [`LKMM`]) against atomic operations on the returned reference. Note that if all other + /// accesses are atomic, then this safety requirement is trivially fulfilled. + /// + /// [`LKMM`]: srctree/tools/memory-model + /// + /// # Examples + /// + /// Using [`Atomic::from_ptr()`] combined with [`Atomic::load()`] or [`Atomic::store()`] can + /// achieve the same functionality as `READ_ONCE()`/`smp_load_acquire()` or + /// `WRITE_ONCE()`/`smp_store_release()` in C side: + /// + /// ``` + /// # use kernel::types::Opaque; + /// use kernel::sync::atomic::{Atomic, Relaxed, Release}; + /// + /// // Assume there is a C struct `foo`. + /// mod cbindings { + /// #[repr(C)] + /// pub(crate) struct foo { + /// pub(crate) a: i32, + /// pub(crate) b: i32 + /// } + /// } + /// + /// let tmp = Opaque::new(cbindings::foo { a: 1, b: 2 }); + /// + /// // struct foo *foo_ptr = ..; + /// let foo_ptr = tmp.get(); + /// + /// // SAFETY: `foo_ptr` is valid, and `.a` is in bounds. + /// let foo_a_ptr = unsafe { &raw mut (*foo_ptr).a }; + /// + /// // a = READ_ONCE(foo_ptr->a); + /// // + /// // SAFETY: `foo_a_ptr` is valid for read, and all other accesses on it is atomic, so no + /// // data race. + /// let a = unsafe { Atomic::from_ptr(foo_a_ptr) }.load(Relaxed); + /// # assert_eq!(a, 1); + /// + /// // smp_store_release(&foo_ptr->a, 2); + /// // + /// // SAFETY: `foo_a_ptr` is valid for writes, and all other accesses on it is atomic, so + /// // no data race. + /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); + /// ``` + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { + // CAST: `T` and `Atomic` have the same size, alignment and bit validity. + // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will + // live long enough. It's safe to return a `&Atomic` because function safety requirement + // guarantees other accesses won't cause data races. + unsafe { &*ptr.cast::() } + } + + /// Returns a pointer to the underlying atomic `T`. + /// + /// Note that use of the return pointer must not cause data races defined by [`LKMM`]. + /// + /// # Guarantees + /// + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::()`]). + /// + /// [`LKMM`]: srctree/tools/memory-model + /// [`align_of::()`]: core::mem::align_of + pub const fn as_ptr(&self) -> *mut T { + // GUARANTEE: Per the function guarantee of `AtomicRepr::as_ptr()`, the `self.0.as_ptr()` + // must be a valid and properly aligned pointer for `T::Repr`, and per the safety guarantee + // of `AtomicType`, it's a valid and properly aligned pointer of `T`. + self.0.as_ptr().cast() + } + + /// Returns a mutable reference to the underlying atomic `T`. + /// + /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let mut atomic_val = Atomic::new(0u32); + /// let val_mut = atomic_val.get_mut(); + /// *val_mut = 101; + /// assert_eq!(101, atomic_val.load(Relaxed)); + /// ``` + pub fn get_mut(&mut self) -> &mut T { + // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of + // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting + // result is a valid pointer of `T`. + // SAFETY: The pointer is valid per the CAST comment above, and the mutable reference + // guarantees exclusive access. + unsafe { &mut *self.0.as_ptr().cast() } + } +} + +impl Atomic +where + T::Repr: AtomicBasicOps, +{ + /// Loads the value from the atomic `T`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42i32); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// let x = Atomic::new(42i64); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_read", "atomic64_read"))] + #[inline(always)] + pub fn load(&self, _: Ordering) -> T { + let v = { + match Ordering::TYPE { + OrderingType::Relaxed => T::Repr::atomic_read(&self.0), + OrderingType::Acquire => T::Repr::atomic_read_acquire(&self.0), + _ => build_error!("Wrong ordering"), + } + }; + + // SAFETY: `v` comes from reading `self.0`, which is a valid `T` per the type invariants. + unsafe { from_repr(v) } + } + + /// Stores a value to the atomic `T`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42i32); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// x.store(43, Relaxed); + /// + /// assert_eq!(43, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_set", "atomic64_set"))] + #[inline(always)] + pub fn store(&self, v: T, _: Ordering) { + let v = into_repr(v); + + // INVARIANT: `v` is a valid `T`, and is stored to `self.0` by `atomic_set*()`. + match Ordering::TYPE { + OrderingType::Relaxed => T::Repr::atomic_set(&self.0, v), + OrderingType::Release => T::Repr::atomic_set_release(&self.0, v), + _ => build_error!("Wrong ordering"), + } + } +} + +impl core::fmt::Debug for Atomic +where + T::Repr: AtomicBasicOps, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(&self.load(Relaxed), f) + } +} + +impl Atomic +where + T::Repr: AtomicExchangeOps, +{ + /// Atomic exchange. + /// + /// Atomically updates `*self` to `v` and returns the old value of `*self`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// assert_eq!(42, x.xchg(52, Acquire)); + /// assert_eq!(52, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))] + #[inline(always)] + pub fn xchg(&self, v: T, _: Ordering) -> T { + let v = into_repr(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to + // `T`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_xchg(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } + + /// Atomic compare and exchange. + /// + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not + /// modified. + /// + /// Compare: The comparison is done via the byte level comparison between `*self` and `old`. + /// + /// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type + /// parameter indicates, and a failed one doesn't provide any ordering, the load part of a + /// failed cmpxchg is a [`Relaxed`] load. + /// + /// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`, + /// otherwise returns `Err(value)`, and `value` is the current value of `*self`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// // Checks whether cmpxchg succeeded. + /// let success = x.cmpxchg(52, 64, Relaxed).is_ok(); + /// # assert!(!success); + /// + /// // Checks whether cmpxchg failed. + /// let failure = x.cmpxchg(52, 64, Relaxed).is_err(); + /// # assert!(failure); + /// + /// // Uses the old value if failed, probably re-try cmpxchg. + /// match x.cmpxchg(52, 64, Relaxed) { + /// Ok(_) => { }, + /// Err(old) => { + /// // do something with `old`. + /// # assert_eq!(old, 42); + /// } + /// } + /// + /// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C. + /// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); + /// # assert_eq!(42, latest); + /// assert_eq!(64, x.load(Relaxed)); + /// ``` + /// + /// [`Relaxed`]: ordering::Relaxed + #[doc(alias( + "atomic_cmpxchg", + "atomic64_cmpxchg", + "atomic_try_cmpxchg", + "atomic64_try_cmpxchg", + "compare_exchange" + ))] + #[inline(always)] + pub fn cmpxchg( + &self, + mut old: T, + new: T, + o: Ordering, + ) -> Result { + // Note on code generation: + // + // try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined, + // the compiler is able to figure out that branch is not needed if the users don't care + // about whether the operation succeeds or not. One exception is on x86, due to commit + // 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the + // atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the + // success of cmpxchg and only wants to use the old value. For example, for code like: + // + // let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); + // + // It will still generate code: + // + // movl $0x40, %ecx + // movl $0x34, %eax + // lock + // cmpxchgl %ecx, 0x4(%rsp) + // jne 1f + // 2: + // ... + // 1: movl %eax, %ecx + // jmp 2b + // + // This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old" + // location in the C function is always safe to write. + if self.try_cmpxchg(&mut old, new, o) { + Ok(old) + } else { + Err(old) + } + } + + /// Atomic compare and exchange and returns whether the operation succeeds. + /// + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not + /// modified, `*old` is updated to the current value of `*self`. + /// + /// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`]. + /// + /// Returns `true` means the cmpxchg succeeds otherwise returns `false`. + #[inline(always)] + fn try_cmpxchg(&self, old: &mut T, new: T, _: Ordering) -> bool { + let mut tmp = into_repr(*old); + let new = into_repr(new); + + // INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is + // transmutable to `T`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new), + OrderingType::Acquire => { + T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new) + } + OrderingType::Release => { + T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new) + } + OrderingType::Relaxed => { + T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new) + } + } + }; + + // SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants. + *old = unsafe { from_repr(tmp) }; + + ret + } +} + +impl Atomic +where + T::Repr: AtomicArithmeticOps, +{ + /// Atomic add. + /// + /// Atomically updates `*self` to `(*self).wrapping_add(v)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// x.add(12, Relaxed); + /// + /// assert_eq!(54, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn add(&self, v: Rhs, _: ordering::Relaxed) + where + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_add()` due to safety requirement of + // `AtomicAdd`. + T::Repr::atomic_add(&self.0, v); + } + + /// Atomic fetch and add. + /// + /// Atomically updates `*self` to `(*self).wrapping_add(v)`, and returns the value of `*self` + /// before the update. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_add(12, Acquire)); + /// assert_eq!(54, x.load(Relaxed)); + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_add(12, Full)); + /// assert_eq!(54, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn fetch_add(&self, v: Rhs, _: Ordering) -> T + where + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_add*()` due to safety requirement + // of `AtomicAdd`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_fetch_add(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_fetch_add_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_fetch_add_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_fetch_add_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } + + /// Atomic fetch and subtract. + /// + /// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self` + /// before the update. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Acquire)); + /// assert_eq!(30, x.load(Relaxed)); + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Full)); + /// assert_eq!(30, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn fetch_sub(&self, v: Rhs, _: Ordering) -> T + where + // Types that support addition also support subtraction. + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement + // of `AtomicAdd`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } +} + +#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] +#[repr(C)] +#[derive(Clone, Copy)] +struct Flag { + bool_field: bool, +} + +/// # Invariants +/// +/// `padding` must be all zeroes. +#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] +#[repr(C, align(4))] +#[derive(Clone, Copy)] +struct Flag { + #[cfg(target_endian = "big")] + padding: [u8; 3], + bool_field: bool, + #[cfg(target_endian = "little")] + padding: [u8; 3], +} + +impl Flag { + #[inline(always)] + const fn new(b: bool) -> Self { + // INVARIANT: `padding` is all zeroes. + Self { + bool_field: b, + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + padding: [0; 3], + } + } +} + +// SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip +// transmutable to the selected representation (`i8` or `i32`). +unsafe impl AtomicType for Flag { + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] + type Repr = i8; + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + type Repr = i32; +} + +/// An atomic flag type intended to be backed by performance-optimal integer type. +/// +/// The backing integer type is an implementation detail; it may vary by architecture and change +/// in the future. +/// +/// [`AtomicFlag`] is generally preferable to [`Atomic`] when you need read-modify-write +/// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic`] does +/// not save memory due to padding. On some architectures that do not support byte-sized atomic +/// RMW operations, RMW operations on [`Atomic`] are slower. +/// +/// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic`] is fine. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::atomic::{AtomicFlag, Relaxed}; +/// +/// let flag = AtomicFlag::new(false); +/// assert_eq!(false, flag.load(Relaxed)); +/// flag.store(true, Relaxed); +/// assert_eq!(true, flag.load(Relaxed)); +/// ``` +pub struct AtomicFlag(Atomic); + +impl AtomicFlag { + /// Creates a new atomic flag. + #[inline(always)] + pub const fn new(b: bool) -> Self { + Self(Atomic::new(Flag::new(b))) + } + + /// Returns a mutable reference to the underlying flag as a [`bool`]. + /// + /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; + /// + /// let mut atomic_flag = AtomicFlag::new(false); + /// assert_eq!(false, atomic_flag.load(Relaxed)); + /// *atomic_flag.get_mut() = true; + /// assert_eq!(true, atomic_flag.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut bool { + &mut self.0.get_mut().bool_field + } + + /// Loads the value from the atomic flag. + #[inline(always)] + pub fn load(&self, o: Ordering) -> bool { + self.0.load(o).bool_field + } + + /// Stores a value to the atomic flag. + #[inline(always)] + pub fn store(&self, v: bool, o: Ordering) { + self.0.store(Flag::new(v), o); + } + + /// Stores a value to the atomic flag and returns the previous value. + #[inline(always)] + pub fn xchg(&self, new: bool, o: Ordering) -> bool { + self.0.xchg(Flag::new(new), o).bool_field + } + + /// Store a value to the atomic flag if the current value is equal to `old`. + #[inline(always)] + pub fn cmpxchg( + &self, + old: bool, + new: bool, + o: Ordering, + ) -> Result { + match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { + Ok(_) => Ok(old), + Err(f) => Err(f.bool_field), + } + } +} + +/// Atomic load over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic. +#[doc(alias("READ_ONCE", "smp_load_acquire"))] +#[inline(always)] +pub unsafe fn atomic_load( + ptr: *mut T, + o: Ordering, +) -> T +where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent stores from kernel are atomic, hence no data race per + // LKMM. + unsafe { Atomic::from_ptr(ptr) }.load(o) +} + +/// Atomic store over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Release)` maps to `smp_store_release()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("WRITE_ONCE", "smp_store_release"))] +#[inline(always)] +pub unsafe fn atomic_store( + ptr: *mut T, + v: T, + o: Ordering, +) where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.store(v, o); +} + +/// Atomic exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[inline(always)] +pub unsafe fn xchg( + ptr: *mut T, + new: T, + o: Ordering, +) -> T +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.xchg(new, o) +} + +/// Atomic compare and exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("try_cmpxchg"))] +#[inline(always)] +pub unsafe fn cmpxchg( + ptr: *mut T, + old: T, + new: T, + o: Ordering, +) -> Result +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o) +} diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs new file mode 100644 index 000000000..9c8a7a203 --- /dev/null +++ b/rust/kernel/sync/atomic/internal.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic internal implementations. +//! +//! Provides 1:1 mapping to the C atomic operations. + +use crate::{ + bindings, + build_assert::static_assert, + macros::paste, // +}; +use core::cell::UnsafeCell; +use ffi::c_void; + +mod private { + /// Sealed trait marker to disable customized impls on atomic implementation traits. + pub trait Sealed {} +} + +// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`), +// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of +// lower-level C primitives. +impl private::Sealed for i8 {} +impl private::Sealed for i16 {} +impl private::Sealed for *const c_void {} +impl private::Sealed for i32 {} +impl private::Sealed for i64 {} + +/// A marker trait for types that implement atomic operations with C side primitives. +/// +/// This trait is sealed, and only types that map directly to the C side atomics +/// or can be implemented with lower-level C primitives are allowed to implement this: +/// +/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives. +/// - `i32` map to `atomic_t` +/// - `i64` map to `atomic64_t` +pub trait AtomicImpl: Sized + Copy + private::Sealed { + /// The type of the delta in arithmetic or logical operations. + /// + /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of + /// [`Self`], but it may be different for the atomic pointer type. + type Delta; +} + +// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()` +// hence the atomicity is only guaranteed against read-modify-write operations if the architecture +// supports native atomic RmW. +// +// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the +// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to +// be added. +static_assert!( + cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), + "The current implementation of atomic i8/i16/ptr relies on the architecure being \ + ARCH_SUPPORTS_ATOMIC_RMW" +); + +impl AtomicImpl for i8 { + type Delta = Self; +} + +impl AtomicImpl for i16 { + type Delta = Self; +} + +impl AtomicImpl for *const c_void { + type Delta = isize; +} + +// `atomic_t` implements atomic operations on `i32`. +impl AtomicImpl for i32 { + type Delta = Self; +} + +// `atomic64_t` implements atomic operations on `i64`. +impl AtomicImpl for i64 { + type Delta = Self; +} + +/// Atomic representation. +#[repr(transparent)] +pub struct AtomicRepr(UnsafeCell); + +impl AtomicRepr { + /// Creates a new atomic representation `T`. + pub const fn new(v: T) -> Self { + Self(UnsafeCell::new(v)) + } + + /// Returns a pointer to the underlying `T`. + /// + /// # Guarantees + /// + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::()`]). + pub const fn as_ptr(&self) -> *mut T { + // GUARANTEE: `self.0` is an `UnsafeCell`, therefore the pointer returned by `.get()` + // must be valid and properly aligned. + self.0.get() + } +} + +// This macro generates the function signature with given argument list and return type. +macro_rules! declare_atomic_method { + ( + $(#[doc=$doc:expr])* + $func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)? + ) => { + paste!( + $(#[doc = $doc])* + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?; + ); + }; + ( + $(#[doc=$doc:expr])* + $func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? + ) => { + paste!( + declare_atomic_method!( + $(#[doc = $doc])* + [< $func _ $variant >]($($arg_sig)*) $(-> $ret)? + ); + ); + + declare_atomic_method!( + $(#[doc = $doc])* + $func [$($rest)*]($($arg_sig)*) $(-> $ret)? + ); + }; + ( + $(#[doc=$doc:expr])* + $func:ident []($($arg_sig:tt)*) $(-> $ret:ty)? + ) => { + declare_atomic_method!( + $(#[doc = $doc])* + $func($($arg_sig)*) $(-> $ret)? + ); + } +} + +// This macro generates the function implementation with given argument list and return type, and it +// will replace "call(...)" expression with "$ctype _ $func" to call the real C function. +macro_rules! impl_atomic_method { + ( + ($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? { + $unsafe:tt { call($($c_arg:expr),*) } + } + ) => { + paste!( + #[inline(always)] + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? { + // TODO: Ideally we want to use the SAFETY comments written at the macro invocation + // (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments + // are just comments, and they are not passed to macros as tokens, therefore we + // cannot use them here. One potential improvement is that if we support using + // attributes as an alternative for SAFETY comments, then we can use that for macro + // generating code. + // + // SAFETY: specified on macro invocation. + $unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) } + } + ); + }; + ( + ($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? { + $unsafe:tt { call($($arg:tt)*) } + } + ) => { + paste!( + impl_atomic_method!( + ($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? { + $unsafe { call($($arg)*) } + } + ); + ); + impl_atomic_method!( + ($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? { + $unsafe { call($($arg)*) } + } + ); + }; + ( + ($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { call($($arg:tt)*) } + } + ) => { + impl_atomic_method!( + ($ctype) $func($($arg_sig)*) $(-> $ret)? { + $unsafe { call($($arg)*) } + } + ); + } +} + +macro_rules! declare_atomic_ops_trait { + ( + $(#[$attr:meta])* $pub:vis trait $ops:ident { + $( + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } + )* + } + ) => { + $(#[$attr])* + $pub trait $ops: AtomicImpl { + $( + declare_atomic_method!( + $(#[doc=$doc])* + $func[$($variant)*]($($arg_sig)*) $(-> $ret)? + ); + )* + } + } +} + +macro_rules! impl_atomic_ops_for_one { + ( + $ty:ty => $ctype:ident, + $(#[$attr:meta])* $pub:vis trait $ops:ident { + $( + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } + )* + } + ) => { + impl $ops for $ty { + $( + impl_atomic_method!( + ($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { + $unsafe { call($($arg)*) } + } + ); + )* + } + } +} + +// Declares $ops trait with methods and implements the trait. +macro_rules! declare_and_impl_atomic_methods { + ( + [ $($map:tt)* ] + $(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* } + ) => { + declare_and_impl_atomic_methods!( + @with_ops_def + [ $($map)* ] + ( $(#[$attr])* $pub trait $ops { $($body)* } ) + ); + }; + + (@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => { + declare_atomic_ops_trait!( $($ops_def)* ); + + declare_and_impl_atomic_methods!( + @munch + [ $($map)* ] + ( $($ops_def)* ) + ); + }; + + (@munch [] ( $($ops_def:tt)* )) => {}; + + (@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => { + impl_atomic_ops_for_one!( + $ty => $ctype, + $($ops_def)* + ); + + declare_and_impl_atomic_methods!( + @munch + [ $($($rest)*)? ] + ( $($ops_def)* ) + ); + }; +} + +declare_and_impl_atomic_methods!( + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] + /// Basic atomic operations + pub trait AtomicBasicOps { + /// Atomic read (load). + fn read[acquire](a: &AtomicRepr) -> Self { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast()) } + } + + /// Atomic set (store). + fn set[release](a: &AtomicRepr, v: Self) { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), v) } + } + } +); + +declare_and_impl_atomic_methods!( + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] + /// Exchange and compare-and-exchange atomic operations + pub trait AtomicExchangeOps { + /// Atomic exchange. + /// + /// Atomically updates `*a` to `v` and returns the old value. + fn xchg[acquire, release, relaxed](a: &AtomicRepr, v: Self) -> Self { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), v) } + } + + /// Atomic compare and exchange. + /// + /// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not + /// modified, `*old` is updated to the current value of `*a`. + /// + /// Return `true` if the update of `*a` occurred, `false` otherwise. + fn try_cmpxchg[acquire, release, relaxed]( + a: &AtomicRepr, old: &mut Self, new: Self + ) -> bool { + // SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)` + // is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) } + } + } +); + +declare_and_impl_atomic_methods!( + [ i32 => atomic, i64 => atomic64 ] + /// Atomic arithmetic operations + pub trait AtomicArithmeticOps { + /// Atomic add (wrapping). + /// + /// Atomically updates `*a` to `(*a).wrapping_add(v)`. + fn add[](a: &AtomicRepr, v: Self::Delta) { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + + /// Atomic fetch and add (wrapping). + /// + /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` + /// before the update. + fn fetch_add[acquire, release, relaxed](a: &AtomicRepr, v: Self::Delta) -> Self { + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + + fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr, v: Self::Delta) -> Self { + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + } +); diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs new file mode 100644 index 000000000..c4e732e72 --- /dev/null +++ b/rust/kernel/sync/atomic/ordering.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory orderings. +//! +//! The semantics of these orderings follows the [`LKMM`] definitions and rules. +//! +//! - [`Acquire`] provides ordering between the load part of the annotated operation and all the +//! following memory accesses, and if there is a store part, the store part has the [`Relaxed`] +//! ordering. +//! - [`Release`] provides ordering between all the preceding memory accesses and the store part of +//! the annotated operation, and if there is a load part, the load part has the [`Relaxed`] +//! ordering. +//! - [`Full`] means "fully-ordered", that is: +//! - It provides ordering between all the preceding memory accesses and the annotated operation. +//! - It provides ordering between the annotated operation and all the following memory accesses. +//! - It provides ordering between all the preceding memory accesses and all the following memory +//! accesses. +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). +//! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are +//! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. +//! +//! [`LKMM`]: srctree/tools/memory-model/ +//! [`explanation`]: srctree/tools/memory-model/Documentation/explanation.txt + +/// The annotation type for relaxed memory ordering, for the description of relaxed memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Relaxed; + +/// The annotation type for acquire memory ordering, for the description of acquire memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Acquire; + +/// The annotation type for release memory ordering, for the description of release memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Release; + +/// The annotation type for fully-ordered memory ordering, for the description fully-ordered memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Full; + +/// Describes the exact memory ordering. +#[doc(hidden)] +pub enum OrderingType { + /// Relaxed ordering. + Relaxed, + /// Acquire ordering. + Acquire, + /// Release ordering. + Release, + /// Fully-ordered. + Full, +} + +mod internal { + /// Sealed trait, can be only implemented inside atomic mod. + pub trait Sealed {} + + impl Sealed for super::Relaxed {} + impl Sealed for super::Acquire {} + impl Sealed for super::Release {} + impl Sealed for super::Full {} +} + +/// The trait bound for annotating operations that support any ordering. +pub trait Ordering: internal::Sealed { + /// Describes the exact memory ordering. + const TYPE: OrderingType; +} + +impl Ordering for Relaxed { + const TYPE: OrderingType = OrderingType::Relaxed; +} + +impl Ordering for Acquire { + const TYPE: OrderingType = OrderingType::Acquire; +} + +impl Ordering for Release { + const TYPE: OrderingType = OrderingType::Release; +} + +impl Ordering for Full { + const TYPE: OrderingType = OrderingType::Full; +} + +/// The trait bound for operations that only support acquire or relaxed ordering. +pub trait AcquireOrRelaxed: Ordering {} + +impl AcquireOrRelaxed for Acquire {} +impl AcquireOrRelaxed for Relaxed {} + +/// The trait bound for operations that only support release or relaxed ordering. +pub trait ReleaseOrRelaxed: Ordering {} + +impl ReleaseOrRelaxed for Release {} +impl ReleaseOrRelaxed for Relaxed {} diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs new file mode 100644 index 000000000..3d63f4079 --- /dev/null +++ b/rust/kernel/sync/atomic/predefine.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Pre-defined atomic types + +use crate::prelude::*; + +// Ensure size and alignment requirements are checked. +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); + +// SAFETY: `bool` has the same size and alignment as `i8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (false) and 1 (true). Those are valid `i8` values, so `bool` is +// round-trip transmutable to `i8`. +unsafe impl super::AtomicType for bool { + type Repr = i8; +} + +// SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i8 { + type Repr = i8; +} + +// SAFETY: `i16` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i16 { + type Repr = i16; +} + +// SAFETY: +// +// - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*mut T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl super::AtomicType for *mut T { + type Repr = *const c_void; +} + +// SAFETY: +// +// - `*const T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*const T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl super::AtomicType for *const T { + type Repr = *const c_void; +} + +// SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i32 { + type Repr = i32; +} + +// SAFETY: The wrapping add result of two `i32`s is a valid `i32`. +unsafe impl super::AtomicAdd for i32 { + fn rhs_into_delta(rhs: i32) -> i32 { + rhs + } +} + +// SAFETY: `i64` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i64 { + type Repr = i64; +} + +// SAFETY: The wrapping add result of two `i64`s is a valid `i64`. +unsafe impl super::AtomicAdd for i64 { + fn rhs_into_delta(rhs: i64) -> i64 { + rhs + } +} + +// Defines an internal type that always maps to the integer type which has the same size alignment +// as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to +// `isize_atomic_repr`, which also always implements `AtomicImpl`. +#[allow(non_camel_case_types)] +#[cfg(not(testlib))] +#[cfg(not(CONFIG_64BIT))] +type isize_atomic_repr = i32; +#[allow(non_camel_case_types)] +#[cfg(not(testlib))] +#[cfg(CONFIG_64BIT)] +type isize_atomic_repr = i64; + +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "32")] +type isize_atomic_repr = i32; +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "64")] +type isize_atomic_repr = i64; + +// Ensure size and alignment requirements are checked. +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); + +// SAFETY: `isize` has the same size and alignment with `isize_atomic_repr`, and is round-trip +// transmutable to `isize_atomic_repr`. +unsafe impl super::AtomicType for isize { + type Repr = isize_atomic_repr; +} + +// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. +unsafe impl super::AtomicAdd for isize { + fn rhs_into_delta(rhs: isize) -> isize_atomic_repr { + rhs as isize_atomic_repr + } +} + +// SAFETY: `u32` and `i32` has the same size and alignment, and `u32` is round-trip transmutable to +// `i32`. +unsafe impl super::AtomicType for u32 { + type Repr = i32; +} + +// SAFETY: The wrapping add result of two `i32`s is a valid `u32`. +unsafe impl super::AtomicAdd for u32 { + fn rhs_into_delta(rhs: u32) -> i32 { + rhs as i32 + } +} + +// SAFETY: `u64` and `i64` has the same size and alignment, and `u64` is round-trip transmutable to +// `i64`. +unsafe impl super::AtomicType for u64 { + type Repr = i64; +} + +// SAFETY: The wrapping add result of two `i64`s is a valid `u64`. +unsafe impl super::AtomicAdd for u64 { + fn rhs_into_delta(rhs: u64) -> i64 { + rhs as i64 + } +} + +// SAFETY: `usize` has the same size and alignment with `isize_atomic_repr`, and is round-trip +// transmutable to `isize_atomic_repr`. +unsafe impl super::AtomicType for usize { + type Repr = isize_atomic_repr; +} + +// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. +unsafe impl super::AtomicAdd for usize { + fn rhs_into_delta(rhs: usize) -> isize_atomic_repr { + rhs as isize_atomic_repr + } +} + +#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)] +#[macros::kunit_tests(rust_atomics)] +mod tests { + use super::super::*; + + // Call $fn($val) with each $type of $val. + macro_rules! for_each_type { + ($val:literal in [$($type:ty),*] $fn:expr) => { + $({ + let v: $type = $val; + + $fn(v); + })* + } + } + + #[test] + fn atomic_basic_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + assert_eq!(v, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); + }); + } + + #[test] + fn atomic_acquire_release_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + + x.store(v, Release); + assert_eq!(v, x.load(Acquire)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + unsafe { atomic_store(ptr, v, Release) }; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); + }); + } + + #[test] + fn atomic_xchg_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + let old = v; + let new = v + 1; + + assert_eq!(old, x.xchg(new, Full)); + assert_eq!(new, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(old, unsafe { xchg(ptr, new, Full) }); + assert_eq!(new, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_cmpxchg_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + let old = v; + let new = v + 1; + + assert_eq!(Err(old), x.cmpxchg(new, new, Full)); + assert_eq!(old, x.load(Relaxed)); + assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); + assert_eq!(new, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); + assert_eq!(old, x.load(Relaxed)); + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); + assert_eq!(new, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_arithmetic_tests() { + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + assert_eq!(v, x.fetch_add(12, Full)); + assert_eq!(v + 12, x.load(Relaxed)); + + x.add(13, Relaxed); + + assert_eq!(v + 25, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_bool_tests() { + let x = Atomic::new(false); + + assert_eq!(false, x.load(Relaxed)); + x.store(true, Relaxed); + assert_eq!(true, x.load(Relaxed)); + + assert_eq!(true, x.xchg(false, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + + assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); + } + + #[test] + fn atomic_ptr_tests() { + let mut v = 42; + let mut u = 43; + let x = Atomic::new(&raw mut v); + + assert_eq!(x.load(Acquire), &raw mut v); + assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v)); + assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v)); + assert_eq!(x.load(Relaxed), &raw mut u); + + let x = Atomic::new(&raw const v); + + assert_eq!(x.load(Acquire), &raw const v); + assert_eq!( + x.cmpxchg(&raw const u, &raw const u, Relaxed), + Err(&raw const v) + ); + assert_eq!( + x.cmpxchg(&raw const v, &raw const u, Relaxed), + Ok(&raw const v) + ); + assert_eq!(x.load(Relaxed), &raw const u); + } + + #[test] + fn atomic_flag_tests() { + let mut flag = AtomicFlag::new(false); + + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(true, flag.load(Relaxed)); + + assert_eq!(true, flag.xchg(false, Relaxed)); + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); + assert_eq!(false, flag.load(Relaxed)); + } +} diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs new file mode 100644 index 000000000..1180695d5 --- /dev/null +++ b/rust/kernel/sync/barrier.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory barriers. +//! +//! These primitives have the same semantics as their C counterparts: and the precise definitions +//! of semantics can be found at [`LKMM`]. +//! +//! [`LKMM`]: srctree/tools/memory-model/ + +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + +/// A compiler barrier. +/// +/// A barrier that prevents compiler from reordering memory accesses across the barrier. +#[inline(always)] +pub(crate) fn barrier() { + // By default, Rust inline asms are treated as being able to access any memory or flags, hence + // it suffices as a compiler barrier. + // + // SAFETY: An empty asm block. + unsafe { core::arch::asm!("") }; +} + +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb(_: T) { + T::run() +} + +/// Memory barrier between CPUs. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb>(_: T) { + if cfg!(CONFIG_SMP) { + T::run() + } else { + barrier() + } +} + +/// Memory barrier between local CPU and bus-mastering devices. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() +} diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs new file mode 100644 index 000000000..35ff049ff --- /dev/null +++ b/rust/kernel/sync/completion.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Completion support. +//! +//! Reference: +//! +//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h) + +use crate::{bindings, prelude::*, types::Opaque}; + +/// Synchronization primitive to signal when a certain task has been completed. +/// +/// The [`Completion`] synchronization primitive signals when a certain task has been completed by +/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, Completion}; +/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +/// +/// #[pin_data] +/// struct MyTask { +/// #[pin] +/// work: Work, +/// #[pin] +/// done: Completion, +/// } +/// +/// impl_has_work! { +/// impl HasWork for MyTask { self.work } +/// } +/// +/// impl MyTask { +/// fn new() -> Result> { +/// let this = Arc::pin_init(pin_init!(MyTask { +/// work <- new_work!("MyTask::work"), +/// done <- Completion::new(), +/// }), GFP_KERNEL)?; +/// +/// let _ = workqueue::system().enqueue(this.clone()); +/// +/// Ok(this) +/// } +/// +/// fn wait_for_completion(&self) { +/// self.done.wait_for_completion(); +/// +/// pr_info!("Completion: task complete\n"); +/// } +/// } +/// +/// impl WorkItem for MyTask { +/// type Pointer = Arc; +/// +/// fn run(this: Arc) { +/// // process this task +/// this.done.complete_all(); +/// } +/// } +/// +/// let task = MyTask::new()?; +/// task.wait_for_completion(); +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data] +pub struct Completion { + #[pin] + inner: Opaque, +} + +// SAFETY: `Completion` is safe to be send to any task. +unsafe impl Send for Completion {} + +// SAFETY: `Completion` is safe to be accessed concurrently. +unsafe impl Sync for Completion {} + +impl Completion { + /// Create an initializer for a new [`Completion`]. + pub fn new() -> impl PinInit { + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::completion| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`. + unsafe { bindings::init_completion(slot) }; + }), + }) + } + + fn as_raw(&self) -> *mut bindings::completion { + self.inner.get() + } + + /// Signal all tasks waiting on this completion. + /// + /// This method wakes up all tasks waiting on this completion; after this operation the + /// completion is permanently done, i.e. signals all current and future waiters. + #[inline] + pub fn complete_all(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::complete_all(self.as_raw()) }; + } + + /// Wait for completion of a task. + /// + /// This method waits for the completion of a task; it is not interruptible and there is no + /// timeout. + /// + /// See also [`Completion::complete_all`]. + #[inline] + pub fn wait_for_completion(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::wait_for_completion(self.as_raw()) }; + } +} diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs new file mode 100644 index 000000000..69d58dfba --- /dev/null +++ b/rust/kernel/sync/condvar.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A condition variable. +//! +//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition +//! variable. + +use super::{lock::Backend, lock::Guard, LockClassKey}; +use crate::{ + ffi::{c_int, c_long}, + str::{CStr, CStrExt as _}, + task::{ + MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE, + }, + time::Jiffies, + types::Opaque, +}; +use core::{marker::PhantomPinned, pin::Pin, ptr}; +use pin_init::{pin_data, pin_init, PinInit}; + +/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_condvar { + ($($name:literal)?) => { + $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_condvar; + +/// A conditional variable. +/// +/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to +/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And +/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or +/// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up +/// spuriously. +/// +/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init!) and [`new_condvar`] macros. +/// +/// # Examples +/// +/// The following is an example of using a condvar with a mutex: +/// +/// ``` +/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex}; +/// +/// #[pin_data] +/// pub struct Example { +/// #[pin] +/// value: Mutex, +/// +/// #[pin] +/// value_changed: CondVar, +/// } +/// +/// /// Waits for `e.value` to become `v`. +/// fn wait_for_value(e: &Example, v: u32) { +/// let mut guard = e.value.lock(); +/// while *guard != v { +/// e.value_changed.wait(&mut guard); +/// } +/// } +/// +/// /// Increments `e.value` and notifies all potential waiters. +/// fn increment(e: &Example) { +/// *e.value.lock() += 1; +/// e.value_changed.notify_all(); +/// } +/// +/// /// Allocates a new boxed `Example`. +/// fn new_example() -> Result>> { +/// KBox::pin_init(pin_init!(Example { +/// value <- new_mutex!(0), +/// value_changed <- new_condvar!(), +/// }), GFP_KERNEL) +/// } +/// ``` +/// +/// [`struct wait_queue_head`]: srctree/include/linux/wait.h +#[pin_data] +pub struct CondVar { + #[pin] + pub(crate) wait_queue_head: Opaque, + + /// A condvar needs to be pinned because it contains a [`struct list_head`] that is + /// self-referential, so it cannot be safely moved once it is initialised. + /// + /// [`struct list_head`]: srctree/include/linux/types.h + #[pin] + _pin: PhantomPinned, +} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. +unsafe impl Send for CondVar {} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads +// concurrently. +unsafe impl Sync for CondVar {} + +impl CondVar { + /// Constructs a new condvar initialiser. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + pin_init!(Self { + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + wait_queue_head <- Opaque::ffi_init(|slot| unsafe { + bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } + + fn wait_internal( + &self, + wait_state: c_int, + guard: &mut Guard<'_, T, B>, + timeout_in_jiffies: c_long, + ) -> c_long { + let wait = Opaque::::uninit(); + + // SAFETY: `wait` points to valid memory. + unsafe { bindings::init_wait(wait.get()) }; + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { + bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state) + }; + + // SAFETY: Switches to another thread. The timeout can be any number. + let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) }); + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) }; + + ret + } + + /// Releases the lock and waits for a notification in uninterruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by + /// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up + /// spuriously. + pub fn wait(&self, guard: &mut Guard<'_, T, B>) { + self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may + /// wake up due to signals. It may also wake up spuriously. + /// + /// Returns whether there is a signal pending. + #[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible(&self, guard: &mut Guard<'_, T, B>) -> bool { + self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + crate::current!().signal_pending() + } + + /// Releases the lock and waits for a notification in interruptible and freezable mode. + /// + /// The process is allowed to be frozen during this sleep. No lock should be held when calling + /// this function, and there is a lockdep assertion for this. Freezing a task that holds a lock + /// can trivially deadlock vs another task that needs that lock to complete before it too can + /// hit freezable. + #[must_use = "wait_interruptible_freezable returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_freezable( + &self, + guard: &mut Guard<'_, T, B>, + ) -> bool { + self.wait_internal( + TASK_INTERRUPTIBLE | TASK_FREEZABLE, + guard, + MAX_SCHEDULE_TIMEOUT, + ); + crate::current!().signal_pending() + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal. + #[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_timeout( + &self, + guard: &mut Guard<'_, T, B>, + jiffies: Jiffies, + ) -> CondVarTimeoutResult { + let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT); + let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies); + + match (res as Jiffies, crate::current!().signal_pending()) { + (jiffies, true) => CondVarTimeoutResult::Signal { jiffies }, + (0, false) => CondVarTimeoutResult::Timeout, + (jiffies, false) => CondVarTimeoutResult::Woken { jiffies }, + } + } + + /// Calls the kernel function to notify the appropriate number of threads. + fn notify(&self, count: c_int) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { + bindings::__wake_up( + self.wait_queue_head.get(), + TASK_NORMAL, + count, + ptr::null_mut(), + ) + }; + } + + /// Calls the kernel function to notify one thread synchronously. + /// + /// This method behaves like `notify_one`, except that it hints to the scheduler that the + /// current thread is about to go to sleep, so it should schedule the target thread on the same + /// CPU. + #[inline] + pub fn notify_sync(&self) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; + } + + /// Wakes a single waiter up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + #[inline] + pub fn notify_one(&self) { + self.notify(1); + } + + /// Wakes all waiters up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + #[inline] + pub fn notify_all(&self) { + self.notify(0); + } +} + +/// The return type of `wait_timeout`. +pub enum CondVarTimeoutResult { + /// The timeout was reached. + Timeout, + /// Somebody woke us up. + Woken { + /// Remaining sleep duration. + jiffies: Jiffies, + }, + /// A signal occurred. + Signal { + /// Remaining sleep duration. + jiffies: Jiffies, + }, +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs new file mode 100644 index 000000000..10b6b5e9b --- /dev/null +++ b/rust/kernel/sync/lock.rs @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic kernel lock and guard. +//! +//! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes, +//! spinlocks, raw spinlocks) to be provided with minimal effort. + +use super::LockClassKey; +use crate::{ + str::{CStr, CStrExt as _}, + types::{NotThreadSafe, Opaque, ScopeGuard}, +}; +use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; +use pin_init::{pin_data, pin_init, PinInit, Wrapper}; + +pub mod mutex; +pub mod spinlock; + +pub(super) mod global; +pub use global::{GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; + +/// The "backend" of a lock. +/// +/// It is the actual implementation of the lock, without the need to repeat patterns used in all +/// locks. +/// +/// # Safety +/// +/// - Implementers must ensure that only one thread/CPU may access the protected data once the lock +/// is owned, that is, between calls to [`lock`] and [`unlock`]. +/// - Implementers must also ensure that [`relock`] uses the same locking method as the original +/// lock operation. +/// +/// [`lock`]: Backend::lock +/// [`unlock`]: Backend::unlock +/// [`relock`]: Backend::relock +pub unsafe trait Backend { + /// The state required by the lock. + type State; + + /// The state required to be kept between [`lock`] and [`unlock`]. + /// + /// [`lock`]: Backend::lock + /// [`unlock`]: Backend::unlock + type GuardState; + + /// Initialises the lock. + /// + /// # Safety + /// + /// `ptr` must be valid for write for the duration of the call, while `name` and `key` must + /// remain valid for read indefinitely. + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ); + + /// Acquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + #[must_use] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; + + /// Tries to acquire the lock. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn try_lock(ptr: *mut Self::State) -> Option; + + /// Releases the lock, giving up its ownership. + /// + /// # Safety + /// + /// It must only be called by the current owner of the lock. + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState); + + /// Reacquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or + /// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now. + unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) { + // SAFETY: The safety requirements ensure that the lock is initialised. + *guard_state = unsafe { Self::lock(ptr) }; + } + + /// Asserts that the lock is held using lockdep. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn assert_is_held(ptr: *mut Self::State); +} + +/// A mutual exclusion primitive. +/// +/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock +/// [`Backend`] specified as the generic parameter `B`. +#[repr(C)] +#[pin_data] +pub struct Lock { + /// The kernel lock object. + #[pin] + state: Opaque, + + /// Some locks are known to be self-referential (e.g., mutexes), while others are architecture + /// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case + /// some architecture uses self-references now or in the future. + #[pin] + _pin: PhantomPinned, + + /// The data protected by the lock. + #[pin] + pub(crate) data: UnsafeCell, +} + +// SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for Lock {} + +// SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl Sync for Lock {} + +impl Lock { + /// Constructs a new lock initialiser. + pub fn new( + t: impl PinInit, + name: &'static CStr, + key: Pin<&'static LockClassKey>, + ) -> impl PinInit { + pin_init!(Self { + data <- UnsafeCell::pin_init(t), + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } +} + +impl Lock<(), B> { + /// Constructs a [`Lock`] from a raw pointer. + /// + /// This can be useful for interacting with a lock which was initialised outside of Rust. + /// + /// # Safety + /// + /// The caller promises that `ptr` points to a valid initialised instance of [`State`] during + /// the whole lifetime of `'a`. + /// + /// [`State`]: Backend::State + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut B::State) -> &'a Self { + // SAFETY: + // - By the safety contract `ptr` must point to a valid initialised instance of `B::State` + // - Since the lock data type is `()` which is a ZST, `state` is the only non-ZST member of + // the struct + // - Combined with `#[repr(C)]`, this guarantees `Self` has an equivalent data layout to + // `B::State`. + unsafe { &*ptr.cast() } + } +} + +impl Lock { + /// Acquires the lock and gives the caller access to the data protected by it. + #[inline] + pub fn lock(&self) -> Guard<'_, T, B> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + let state = unsafe { B::lock(self.state.get()) }; + // SAFETY: The lock was just acquired. + unsafe { Guard::new(self, state) } + } + + /// Tries to acquire the lock. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + // `Option` is not `#[must_use]` even if `T` is, thus the attribute is needed here. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock(&self) -> Option> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + unsafe { B::try_lock(self.state.get()).map(|state| Guard::new(self, state)) } + } +} + +/// A lock guard. +/// +/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock +/// when a guard goes out of scope. It also provides a safe and convenient way to access the data +/// protected by the lock. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, T: ?Sized, B: Backend> { + pub(crate) lock: &'a Lock, + pub(crate) state: B::GuardState, + _not_send: NotThreadSafe, +} + +// SAFETY: `Guard` is sync when the data protected by the lock is also sync. +unsafe impl Sync for Guard<'_, T, B> {} + +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Returns the lock that this guard originates from. + /// + /// # Examples + /// + /// The following example shows how to use [`Guard::lock_ref()`] to assert the corresponding + /// lock is held. + /// + /// ``` + /// # use kernel::{new_spinlock, sync::lock::{Backend, Guard, Lock}}; + /// # use pin_init::stack_pin_init; + /// + /// fn assert_held(guard: &Guard<'_, T, B>, lock: &Lock) { + /// // Address-equal means the same lock. + /// assert!(core::ptr::eq(guard.lock_ref(), lock)); + /// } + /// + /// // Creates a new lock on the stack. + /// stack_pin_init!{ + /// let l = new_spinlock!(42) + /// } + /// + /// let g = l.lock(); + /// + /// // `g` originates from `l`. + /// assert_held(&g, &l); + /// ``` + pub fn lock_ref(&self) -> &'a Lock { + self.lock + } + + pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce() -> U) -> U { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + + let _relock = ScopeGuard::new(|| + // SAFETY: The lock was just unlocked above and is being relocked now. + unsafe { B::relock(self.lock.state.get(), &mut self.state) }); + + cb() + } + + /// Returns a pinned mutable reference to the protected data. + /// + /// The guard implements [`DerefMut`] when `T: Unpin`, so for [`Unpin`] + /// types [`DerefMut`] should be used instead of this function. + /// + /// [`DerefMut`]: core::ops::DerefMut + /// [`Unpin`]: core::marker::Unpin + /// + /// # Examples + /// + /// ``` + /// # use kernel::sync::{Mutex, MutexGuard}; + /// # use core::{pin::Pin, marker::PhantomPinned}; + /// struct Data(PhantomPinned); + /// + /// fn example(mutex: &Mutex) { + /// let mut data: MutexGuard<'_, Data> = mutex.lock(); + /// let mut data: Pin<&mut Data> = data.as_mut(); + /// } + /// ``` + pub fn as_mut(&mut self) -> Pin<&mut T> { + // SAFETY: `self.lock.data` is structurally pinned. + unsafe { Pin::new_unchecked(&mut *self.lock.data.get()) } + } +} + +impl core::ops::Deref for Guard<'_, T, B> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &*self.lock.data.get() } + } +} + +impl core::ops::DerefMut for Guard<'_, T, B> +where + T: Unpin, +{ + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &mut *self.lock.data.get() } + } +} + +impl Drop for Guard<'_, T, B> { + #[inline] + fn drop(&mut self) { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + } +} + +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Constructs a new immutable lock guard. + /// + /// # Safety + /// + /// The caller must ensure that it owns the lock. + #[inline] + pub unsafe fn new(lock: &'a Lock, state: B::GuardState) -> Self { + // SAFETY: The caller can only hold the lock if `Backend::init` has already been called. + unsafe { B::assert_is_held(lock.state.get()) }; + + Self { + lock, + state, + _not_send: NotThreadSafe, + } + } +} diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs new file mode 100644 index 000000000..ebb10521d --- /dev/null +++ b/rust/kernel/sync/lock/global.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Support for defining statics containing locks. + +use crate::{ + str::{CStr, CStrExt as _}, + sync::lock::{Backend, Guard, Lock}, + sync::{LockClassKey, LockedBy}, + types::Opaque, +}; +use core::{ + cell::UnsafeCell, + marker::{PhantomData, PhantomPinned}, + pin::Pin, +}; + +/// Trait implemented for marker types for global locks. +/// +/// See [`global_lock!`] for examples. +pub trait GlobalLockBackend { + /// The name for this global lock. + const NAME: &'static CStr; + /// Item type stored in this global lock. + type Item: 'static; + /// The backend used for this global lock. + type Backend: Backend + 'static; + /// The class for this global lock. + fn get_lock_class() -> Pin<&'static LockClassKey>; +} + +/// Type used for global locks. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLock { + inner: Lock, +} + +impl GlobalLock { + /// Creates a global lock. + /// + /// # Safety + /// + /// * Before any other method on this lock is called, [`Self::init`] must be called. + /// * The type `B` must not be used with any other lock. + pub const unsafe fn new(data: B::Item) -> Self { + Self { + inner: Lock { + state: Opaque::uninit(), + data: UnsafeCell::new(data), + _pin: PhantomPinned, + }, + } + } + + /// Initializes a global lock. + /// + /// # Safety + /// + /// Must not be called more than once on a given lock. + pub unsafe fn init(&'static self) { + // SAFETY: The pointer to `state` is valid for the duration of this call, and both `name` + // and `key` are valid indefinitely. The `state` is pinned since we have a `'static` + // reference to `self`. + // + // We have exclusive access to the `state` since the caller of `new` promised to call + // `init` before using any other methods. As `init` can only be called once, all other + // uses of this lock must happen after this call. + unsafe { + B::Backend::init( + self.inner.state.get(), + B::NAME.as_char_ptr(), + B::get_lock_class().as_ptr(), + ) + } + } + + /// Lock this global lock. + #[inline] + pub fn lock(&'static self) -> GlobalGuard { + GlobalGuard { + inner: self.inner.lock(), + } + } + + /// Try to lock this global lock. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock(&'static self) -> Option> { + Some(GlobalGuard { + inner: self.inner.try_lock()?, + }) + } +} + +/// A guard for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct GlobalGuard { + inner: Guard<'static, B::Item, B::Backend>, +} + +impl core::ops::Deref for GlobalGuard { + type Target = B::Item; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl core::ops::DerefMut for GlobalGuard +where + B::Item: Unpin, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +/// A version of [`LockedBy`] for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLockedBy { + _backend: PhantomData, + value: UnsafeCell, +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl Send for GlobalLockedBy +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy: Send, +{ +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl Sync for GlobalLockedBy +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy: Sync, +{ +} + +impl GlobalLockedBy { + /// Create a new [`GlobalLockedBy`]. + /// + /// The provided value will be protected by the global lock indicated by `B`. + pub fn new(val: T) -> Self { + Self { + value: UnsafeCell::new(val), + _backend: PhantomData, + } + } +} + +impl GlobalLockedBy { + /// Access the value immutably. + /// + /// The caller must prove shared access to the lock. + pub fn as_ref<'a>(&'a self, _guard: &'a GlobalGuard) -> &'a T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &*self.value.get() } + } + + /// Access the value mutably. + /// + /// The caller must prove shared exclusive to the lock. + pub fn as_mut<'a>(&'a self, _guard: &'a mut GlobalGuard) -> &'a mut T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &mut *self.value.get() } + } + + /// Access the value mutably directly. + /// + /// The caller has exclusive access to this `GlobalLockedBy`, so they do not need to hold the + /// lock. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } +} + +/// Defines a global lock. +/// +/// The global mutex must be initialized before first use. Usually this is done by calling +/// [`GlobalLock::init`] in the module initializer. +/// +/// # Examples +/// +/// A global counter: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_COUNTER: Mutex = 0; +/// } +/// +/// fn increment_counter() -> u32 { +/// let mut guard = MY_COUNTER.lock(); +/// *guard += 1; +/// *guard +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// // SAFETY: Called exactly once. +/// unsafe { MY_COUNTER.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +/// +/// A global mutex used to protect all instances of a given struct: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// use kernel::sync::{GlobalGuard, GlobalLockedBy}; +/// +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_MUTEX: Mutex<()> = (); +/// } +/// +/// /// All instances of this struct are protected by `MY_MUTEX`. +/// struct MyStruct { +/// my_counter: GlobalLockedBy, +/// } +/// +/// impl MyStruct { +/// /// Increment the counter in this instance. +/// /// +/// /// The caller must hold the `MY_MUTEX` mutex. +/// fn increment(&self, guard: &mut GlobalGuard) -> u32 { +/// let my_counter = self.my_counter.as_mut(guard); +/// *my_counter += 1; +/// *my_counter +/// } +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// // SAFETY: Called exactly once. +/// unsafe { MY_MUTEX.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +#[macro_export] +macro_rules! global_lock { + { + $(#[$meta:meta])* $pub:vis + unsafe(uninit) static $name:ident: $kind:ident<$valuety:ty> = $value:expr; + } => { + #[doc = ::core::concat!( + "Backend type used by [`", + ::core::stringify!($name), + "`](static@", + ::core::stringify!($name), + ")." + )] + #[allow(non_camel_case_types, unreachable_pub)] + $pub enum $name {} + + impl $crate::sync::lock::GlobalLockBackend for $name { + const NAME: &'static $crate::str::CStr = $crate::c_str!(::core::stringify!($name)); + type Item = $valuety; + type Backend = $crate::global_lock_inner!(backend $kind); + + fn get_lock_class() -> Pin<&'static $crate::sync::LockClassKey> { + $crate::static_lock_class!() + } + } + + $(#[$meta])* + $pub static $name: $crate::sync::lock::GlobalLock<$name> = { + // Defined here to be outside the unsafe scope. + let init: $valuety = $value; + + // SAFETY: + // * The user of this macro promises to initialize the macro before use. + // * We are only generating one static with this backend type. + unsafe { $crate::sync::lock::GlobalLock::new(init) } + }; + }; +} +pub use global_lock; + +#[doc(hidden)] +#[macro_export] +macro_rules! global_lock_inner { + (backend Mutex) => { + $crate::sync::lock::mutex::MutexBackend + }; + (backend SpinLock) => { + $crate::sync::lock::spinlock::SpinLockBackend + }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; +} diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs new file mode 100644 index 000000000..cda0203ef --- /dev/null +++ b/rust/kernel/sync/lock/mutex.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel mutex. +//! +//! This module allows Rust code to use the kernel's `struct mutex`. + +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_mutex; + +/// A mutual exclusion primitive. +/// +/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, +/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is +/// unlocked, at which point another thread will be allowed to wake up and make progress. +/// +/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. +/// +/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_mutex`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a mutex. +/// +/// ``` +/// use kernel::sync::{new_mutex, Mutex}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: Mutex, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_mutex!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a mutex despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::Mutex; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &Mutex) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`struct mutex`]: srctree/include/linux/mutex.h +pub type Mutex = super::Lock; + +/// A [`Guard`] acquired from locking a [`Mutex`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`Mutex`]. It will unlock +/// the [`Mutex`] upon being dropped. +/// +/// [`Guard`]: super::Guard +pub type MutexGuard<'a, T> = super::Guard<'a, T, MutexBackend>; + +/// A kernel `struct mutex` lock backend. +pub struct MutexBackend; + +// SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion. +unsafe impl super::Backend for MutexBackend { + type State = bindings::mutex; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__mutex_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::mutex_lock(ptr) }; + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the mutex. + unsafe { bindings::mutex_unlock(ptr) }; + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::mutex_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::mutex_assert_is_held(ptr) } + } +} diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs new file mode 100644 index 000000000..aafc80125 --- /dev/null +++ b/rust/kernel/sync/lock/spinlock.rs @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel spinlock. +//! +//! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; + +/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLock::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock; + +/// A spinlock. +/// +/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only +/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is +/// unlocked, at which point another CPU will be allowed to make progress. +/// +/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a spinlock. +/// +/// ``` +/// use kernel::sync::{new_spinlock, SpinLock}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: SpinLock, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_spinlock!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a spinlock despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::SpinLock; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &SpinLock) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`spinlock_t`]: srctree/include/linux/spinlock.h +pub type SpinLock = Lock; + +/// A kernel `spinlock_t` lock backend. +pub struct SpinLockBackend; + +/// A [`Guard`] acquired from locking a [`SpinLock`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock +/// the [`SpinLock`] upon being dropped. +pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. +/// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and access a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be disabled already. +/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount +/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrq = super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer = Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state = this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting for + state.value += 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata = Test::new()?; + + let _ = workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state = testdata.state.lock(); + state.waiter_ready = true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs new file mode 100644 index 000000000..fb4a1430b --- /dev/null +++ b/rust/kernel/sync/locked_by.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A wrapper for data protected by a lock that does not wrap it. + +use super::{lock::Backend, lock::Lock}; +use crate::build_assert::build_assert; +use core::{cell::UnsafeCell, mem::size_of, ptr}; + +/// Allows access to some data to be serialised by a lock that does not wrap it. +/// +/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., +/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible. +/// For example, if a container has a lock and some data in the contained elements needs +/// to be protected by the same lock. +/// +/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it +/// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence +/// refers to the wrong instance of the lock. +/// +/// [`Mutex`]: super::Mutex +/// [`SpinLock`]: super::SpinLock +/// +/// # Examples +/// +/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an +/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in +/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at +/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also +/// locked; we enforce at run time that the right `InnerDirectory` is locked. +/// +/// ``` +/// use kernel::sync::{LockedBy, Mutex}; +/// +/// struct InnerFile { +/// bytes_used: u64, +/// } +/// +/// struct File { +/// _ino: u32, +/// inner: LockedBy, +/// } +/// +/// struct InnerDirectory { +/// /// The sum of the bytes used by all files. +/// bytes_used: u64, +/// _files: KVec, +/// } +/// +/// struct Directory { +/// _ino: u32, +/// inner: Mutex, +/// } +/// +/// /// Prints `bytes_used` from both the directory and file. +/// fn print_bytes_used(dir: &Directory, file: &File) { +/// let guard = dir.inner.lock(); +/// let inner_file = file.inner.access(&guard); +/// pr_info!("{} {}\n", guard.bytes_used, inner_file.bytes_used); +/// } +/// +/// /// Increments `bytes_used` for both the directory and file. +/// fn inc_bytes_used(dir: &Directory, file: &File) { +/// let mut guard = dir.inner.lock(); +/// guard.bytes_used += 10; +/// +/// let file_inner = file.inner.access_mut(&mut guard); +/// file_inner.bytes_used += 10; +/// } +/// +/// /// Creates a new file. +/// fn new_file(ino: u32, dir: &Directory) -> File { +/// File { +/// _ino: ino, +/// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }), +/// } +/// } +/// ``` +pub struct LockedBy { + owner: *const U, + data: UnsafeCell, +} + +// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for LockedBy {} + +// SAFETY: If `T` is not `Sync`, then parallel shared access to this `LockedBy` allows you to use +// `access_mut` to hand out `&mut T` on one thread at the time. The requirement that `T: Send` is +// sufficient to allow that. +// +// If `T` is `Sync`, then the `access` method also becomes available, which allows you to obtain +// several `&T` from several threads at once. However, this is okay as `T` is `Sync`. +unsafe impl Sync for LockedBy {} + +impl LockedBy { + /// Constructs a new instance of [`LockedBy`]. + /// + /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure + /// that the right owner is being used to access the protected data. If the owner is freed, the + /// data becomes inaccessible; if another instance of the owner is allocated *on the same + /// memory location*, the data becomes accessible again: none of this affects memory safety + /// because in any case at most one thread (or CPU) can access the protected data at a time. + pub fn new(owner: &Lock, data: T) -> Self { + build_assert!( + size_of::>() > 0, + "The lock type cannot be a ZST because it may be impossible to distinguish instances" + ); + Self { + owner: owner.data.get(), + data: UnsafeCell::new(data), + } + } +} + +impl LockedBy { + /// Returns a reference to the protected data when the caller provides evidence (via a + /// reference) that the owner is locked. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches + /// the data protected by the lock without actually holding it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access<'a>(&'a self, owner: &'a U) -> &'a T + where + T: Sync, + { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there are only shared references to the owner for the + // duration of 'a, so it's not possible to use `Self::access_mut` to obtain a mutable + // reference to the inner value that aliases with this shared reference. The type is `Sync` + // so there are no other requirements. + unsafe { &*self.data.get() } + } + + /// Returns a mutable reference to the protected data when the caller provides evidence (via a + /// mutable owner) that the owner is locked mutably. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that + /// matches the data protected by the lock without actually holding it. + /// + /// Showing a mutable reference to the owner is sufficient because we know no other references + /// can exist to it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there is only one reference to the owner. + unsafe { &mut *self.data.get() } + } +} diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs new file mode 100644 index 000000000..dc40bfaa5 --- /dev/null +++ b/rust/kernel/sync/poll.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Utilities for working with `struct poll_table`. + +use crate::{ + alloc::AllocError, + bindings, + fs::File, + prelude::*, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // +}; + +/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_poll_condvar { + ($($name:literal)?) => { + $crate::sync::poll::PollCondVar::new( + $crate::optional_name!($($name)?), $crate::static_lock_class!() + ) + }; +} + +/// Wraps the kernel's `poll_table`. +/// +/// # Invariants +/// +/// The pointer must be null or reference a valid `poll_table`. +#[repr(transparent)] +pub struct PollTable<'a> { + table: *mut bindings::poll_table, + _lifetime: PhantomData<&'a bindings::poll_table>, +} + +impl<'a> PollTable<'a> { + /// Creates a [`PollTable`] from a valid pointer. + /// + /// # Safety + /// + /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`. + pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self { + // INVARIANTS: The safety requirements are the same as the struct invariants. + PollTable { + table, + _lifetime: PhantomData, + } + } + + /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified + /// using the condition variable. + pub fn register_wait(&self, file: &File, cv: &PollCondVar) { + // SAFETY: + // * `file.as_ptr()` references a valid file for the duration of this call. + // * `self.table` is null or references a valid poll_table for the duration of this call. + // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory + // containing `cv.wait_queue_head` is invalidated. Since the destructor clears all + // waiters and then waits for an rcu grace period, it's guaranteed that + // `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal + // of the last waiter. + unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) } + } +} + +/// A wrapper around [`CondVar`] that makes it usable with [`PollTable`]. +/// +/// [`CondVar`]: crate::sync::CondVar +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct PollCondVar { + #[pin] + inner: CondVar, +} + +impl PollCondVar { + /// Constructs a new condvar initialiser. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + pin_init!(Self { + inner <- CondVar::new(name, key), + }) + } +} + +// Make the `CondVar` methods callable on `PollCondVar`. +impl Deref for PollCondVar { + type Target = CondVar; + + fn deref(&self) -> &CondVar { + &self.inner + } +} + +#[pinned_drop] +impl PinnedDrop for PollCondVar { + #[inline] + fn drop(self: Pin<&mut Self>) { + // Clear anything registered using `register_wait`. + // + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; + + // Wait for epoll items to be properly removed. + synchronize_rcu(); + } +} + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result { + let b = KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target = PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual destructor of the type, + // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of + // `synchronize_rcu()`. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs new file mode 100644 index 000000000..0daa1ac87 --- /dev/null +++ b/rust/kernel/sync/rcu.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RCU support. +//! +//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h) + +use crate::{bindings, types::NotThreadSafe}; + +/// Evidence that the RCU read side lock is held on the current thread/CPU. +/// +/// The type is explicitly not `Send` because this property is per-thread/CPU. +/// +/// # Invariants +/// +/// The RCU read side lock is actually held while instances of this guard exist. +pub struct Guard(NotThreadSafe); + +impl Guard { + /// Acquires the RCU read side lock and returns a guard. + #[inline] + pub fn new() -> Self { + // SAFETY: An FFI call with no additional requirements. + unsafe { bindings::rcu_read_lock() }; + // INVARIANT: The RCU read side lock was just acquired above. + Self(NotThreadSafe) + } + + /// Explicitly releases the RCU read side lock. + #[inline] + pub fn unlock(self) {} +} + +impl Default for Guard { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl Drop for Guard { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, the RCU read side is locked, so it is ok to unlock it. + unsafe { bindings::rcu_read_unlock() }; + } +} + +/// Acquires the RCU read side lock. +#[inline] +pub fn read_lock() -> Guard { + Guard::new() +} + +/// Wait until all in-flight `call_rcu()` callbacks complete. +/// +/// Note that this primitive does not necessarily wait for an RCU grace period +/// to complete. For example, if there are no RCU callbacks queued anywhere +/// in the system, then [`rcu_barrier()`] is within its rights to return +/// immediately, without waiting for anything, much less an RCU grace period. +/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods +/// beyond those that were already destined to be executed. +/// +/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all +/// pending lazy RCU callbacks. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn rcu_barrier() { + // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period. + unsafe { bindings::rcu_barrier() }; +} + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs new file mode 100644 index 000000000..23a5d201f --- /dev/null +++ b/rust/kernel/sync/refcount.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic reference counting. +//! +//! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h) + +use crate::{ + build_assert::build_assert, + sync::atomic::Atomic, + types::Opaque, // +}; + +/// Atomic reference counter. +/// +/// This type is conceptually an atomic integer, but provides saturation semantics compared to +/// normal atomic integers. Values in the negative range when viewed as a signed integer are +/// saturation (bad) values. For details about the saturation semantics, please refer to top of +/// [`include/linux/refcount.h`](srctree/include/linux/refcount.h). +/// +/// Wraps the kernel's C `refcount_t`. +#[repr(transparent)] +pub struct Refcount(Opaque); + +impl Refcount { + /// Construct a new [`Refcount`] from an initial value. + /// + /// The initial value should be non-saturated. + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub fn new(value: i32) -> Self { + build_assert!(value >= 0, "initial value saturated"); + // SAFETY: There are no safety requirements for this FFI call. + Self(Opaque::new(unsafe { bindings::REFCOUNT_INIT(value) })) + } + + #[inline] + fn as_ptr(&self) -> *mut bindings::refcount_t { + self.0.get() + } + + /// Get the underlying atomic counter that backs the refcount. + /// + /// NOTE: Usage of this function is discouraged as it can circumvent the protections offered by + /// `refcount.h`. If there is no way to achieve the result using APIs in `refcount.h`, then + /// this function can be used. Otherwise consider adding a binding for the required API. + #[inline] + pub fn as_atomic(&self) -> &Atomic { + let ptr = self.0.get().cast(); + // SAFETY: `refcount_t` is a transparent wrapper of `atomic_t`, which is an atomic 32-bit + // integer that is layout-wise compatible with `Atomic`. All values are valid for + // `refcount_t`, despite some of the values being considered saturated and "bad". + unsafe { &*ptr } + } + + /// Set a refcount's value. + #[inline] + pub fn set(&self, value: i32) { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_set(self.as_ptr(), value) } + } + + /// Increment a refcount. + /// + /// It will saturate if overflows and `WARN`. It will also `WARN` if the refcount is 0, as this + /// represents a possible use-after-free condition. + /// + /// Provides no memory ordering, it is assumed that caller already has a reference on the + /// object. + #[inline] + pub fn inc(&self) { + // SAFETY: self is valid. + unsafe { bindings::refcount_inc(self.as_ptr()) } + } + + /// Decrement a refcount. + /// + /// It will `WARN` on underflow and fail to decrement when saturated. + /// + /// Provides release memory ordering, such that prior loads and stores are done + /// before. + #[inline] + pub fn dec(&self) { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_dec(self.as_ptr()) } + } + + /// Decrement a refcount and test if it is 0. + /// + /// It will `WARN` on underflow and fail to decrement when saturated. + /// + /// Provides release memory ordering, such that prior loads and stores are done + /// before, and provides an acquire ordering on success such that memory deallocation + /// must come after. + /// + /// Returns true if the resulting refcount is 0, false otherwise. + /// + /// # Notes + /// + /// A common pattern of using `Refcount` is to free memory when the reference count reaches + /// zero. This means that the reference to `Refcount` could become invalid after calling this + /// function. This is fine as long as the reference to `Refcount` is no longer used when this + /// function returns `false`. It is not necessary to use raw pointers in this scenario, see + /// . + #[inline] + #[must_use = "use `dec` instead if you do not need to test if it is 0"] + pub fn dec_and_test(&self) -> bool { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_dec_and_test(self.as_ptr()) } + } +} + +// SAFETY: `refcount_t` is thread-safe. +unsafe impl Send for Refcount {} + +// SAFETY: `refcount_t` is thread-safe. +unsafe impl Sync for Refcount {} diff --git a/rust/kernel/sync/set_once.rs b/rust/kernel/sync/set_once.rs new file mode 100644 index 000000000..139cef05e --- /dev/null +++ b/rust/kernel/sync/set_once.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A container that can be initialized at most once. + +use super::atomic::{ + ordering::{Acquire, Relaxed, Release}, + Atomic, +}; +use core::{cell::UnsafeCell, mem::MaybeUninit}; + +/// A container that can be populated at most once. Thread safe. +/// +/// Once the a [`SetOnce`] is populated, it remains populated by the same object for the +/// lifetime `Self`. +/// +/// # Invariants +/// +/// - `init` may only increase in value. +/// - `init` may only assume values in the range `0..=2`. +/// - `init == 0` if and only if `value` is uninitialized. +/// - `init == 1` if and only if there is exactly one thread with exclusive +/// access to `self.value`. +/// - `init == 2` if and only if `value` is initialized and valid for shared +/// access. +/// +/// # Example +/// +/// ``` +/// # use kernel::sync::SetOnce; +/// let value = SetOnce::new(); +/// assert_eq!(None, value.as_ref()); +/// +/// let status = value.populate(42u8); +/// assert_eq!(true, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// +/// let status = value.populate(101u8); +/// assert_eq!(false, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// ``` +pub struct SetOnce { + init: Atomic, + value: UnsafeCell>, +} + +impl Default for SetOnce { + fn default() -> Self { + Self::new() + } +} + +impl SetOnce { + /// Create a new [`SetOnce`]. + /// + /// The returned instance will be empty. + pub const fn new() -> Self { + // INVARIANT: The container is empty and we initialize `init` to `0`. + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + init: Atomic::new(0), + } + } + + /// Get a reference to the contained object. + /// + /// Returns [`None`] if this [`SetOnce`] is empty. + pub fn as_ref(&self) -> Option<&T> { + if self.init.load(Acquire) == 2 { + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // is initialized and valid for shared access. + Some(unsafe { &*self.value.get().cast() }) + } else { + None + } + } + + /// Populate the [`SetOnce`]. + /// + /// Returns `true` if the [`SetOnce`] was successfully populated. + pub fn populate(&self, value: T) -> bool { + // INVARIANT: If the swap succeeds: + // - We increase `init`. + // - We write the valid value `1` to `init`. + // - Only one thread can succeed in this write, so we have exclusive access after the + // write. + if let Ok(0) = self.init.cmpxchg(0, 1, Relaxed) { + // SAFETY: By the type invariants of `Self`, the fact that we succeeded in writing `1` + // to `self.init` means we obtained exclusive access to `self.value`. + unsafe { core::ptr::write(self.value.get().cast(), value) }; + // INVARIANT: + // - We increase `init`. + // - We write the valid value `2` to `init`. + // - We release our exclusive access to `self.value` and it is now valid for shared + // access. + self.init.store(2, Release); + true + } else { + false + } + } + + /// Get a copy of the contained object. + /// + /// Returns [`None`] if the [`SetOnce`] is empty. + pub fn copy(&self) -> Option + where + T: Copy, + { + self.as_ref().copied() + } +} + +impl Drop for SetOnce { + fn drop(&mut self) { + if *self.init.get_mut() == 2 { + let value = self.value.get_mut(); + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // contains a valid value. We have exclusive access, as we hold a `mut` reference to + // `self`. + unsafe { value.assume_init_drop() }; + } + } +} + +// SAFETY: `SetOnce` can be transferred across thread boundaries iff the data it contains can. +unsafe impl Send for SetOnce {} + +// SAFETY: `SetOnce` synchronises access to the inner value via atomic operations, +// so shared references are safe when `T: Sync`. Since the inner `T` may be dropped +// on any thread, we also require `T: Send`. +unsafe impl Sync for SetOnce {} diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000..723e5e277 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs new file mode 100644 index 000000000..3336df493 --- /dev/null +++ b/rust/kernel/task.rs @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Tasks (threads and processes). +//! +//! C header: [`include/linux/sched.h`](srctree/include/linux/sched.h). + +use crate::{ + bindings, + mm::MmWithUser, + pid_namespace::PidNamespace, + prelude::*, + sync::aref::ARef, + types::{NotThreadSafe, Opaque}, +}; +use core::{ + ops::Deref, + ptr, // +}; + +/// A sentinel value used for infinite timeouts. +pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX; + +/// Bitmask for tasks that are sleeping in an interruptible state. +pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in an uninterruptible state. +pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in a freezable state. +pub const TASK_FREEZABLE: c_int = bindings::TASK_FREEZABLE as c_int; +/// Convenience constant for waking up tasks regardless of whether they are in interruptible or +/// uninterruptible sleep. +pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint; + +/// Returns the currently running task. +#[macro_export] +macro_rules! current { + () => { + // SAFETY: This expression creates a temporary value that is dropped at the end of the + // caller's scope. The following mechanisms ensure that the resulting `&CurrentTask` cannot + // leave current task context: + // + // * To return to userspace, the caller must leave the current scope. + // * Operations such as `begin_new_exec()` are necessarily unsafe and the caller of + // `begin_new_exec()` is responsible for safety. + // * Rust abstractions for things such as a `kthread_use_mm()` scope must require the + // closure to be `Send`, so the `NotThreadSafe` field of `CurrentTask` ensures that the + // `&CurrentTask` cannot cross the scope in either direction. + unsafe { &*$crate::task::Task::current() } + }; +} + +/// Wraps the kernel's `struct task_struct`. +/// +/// # Invariants +/// +/// All instances are valid tasks created by the C portion of the kernel. +/// +/// Instances of this type are always refcounted, that is, a call to `get_task_struct` ensures +/// that the allocation remains valid at least until the matching call to `put_task_struct`. +/// +/// # Examples +/// +/// The following is an example of getting the PID of the current thread with zero additional cost +/// when compared to the C version: +/// +/// ``` +/// let pid = current!().pid(); +/// ``` +/// +/// Getting the PID of the current process, also zero additional cost: +/// +/// ``` +/// let pid = current!().group_leader().pid(); +/// ``` +/// +/// Getting the current task and storing it in some struct. The reference count is automatically +/// incremented when creating `State` and decremented when it is dropped: +/// +/// ``` +/// use kernel::{task::Task, sync::aref::ARef}; +/// +/// struct State { +/// creator: ARef, +/// index: u32, +/// } +/// +/// impl State { +/// fn new() -> Self { +/// Self { +/// creator: ARef::from(&**current!()), +/// index: 0, +/// } +/// } +/// } +/// ``` +#[repr(transparent)] +pub struct Task(pub(crate) Opaque); + +// SAFETY: By design, the only way to access a `Task` is via the `current` function or via an +// `ARef` obtained through the `AlwaysRefCounted` impl. This means that the only situation in +// which a `Task` can be accessed mutably is when the refcount drops to zero and the destructor +// runs. It is safe for that to happen on any thread, so it is ok for this type to be `Send`. +unsafe impl Send for Task {} + +// SAFETY: It's OK to access `Task` through shared references from other threads because we're +// either accessing properties that don't change (e.g., `pid`, `group_leader`) or that are properly +// synchronised by C code (e.g., `signal_pending`). +unsafe impl Sync for Task {} + +/// Represents the [`Task`] in the `current` global. +/// +/// This type exists to provide more efficient operations that are only valid on the current task. +/// For example, to retrieve the pid-namespace of a task, you must use rcu protection unless it is +/// the current task. +/// +/// # Invariants +/// +/// Each value of this type must only be accessed from the task context it was created within. +/// +/// Of course, every thread is in a different task context, but for the purposes of this invariant, +/// these operations also permanently leave the task context: +/// +/// * Returning to userspace from system call context. +/// * Calling `release_task()`. +/// * Calling `begin_new_exec()` in a binary format loader. +/// +/// Other operations temporarily create a new sub-context: +/// +/// * Calling `kthread_use_mm()` creates a new context, and `kthread_unuse_mm()` returns to the +/// old context. +/// +/// This means that a `CurrentTask` obtained before a `kthread_use_mm()` call may be used again +/// once `kthread_unuse_mm()` is called, but it must not be used between these two calls. +/// Conversely, a `CurrentTask` obtained between a `kthread_use_mm()`/`kthread_unuse_mm()` pair +/// must not be used after `kthread_unuse_mm()`. +#[repr(transparent)] +pub struct CurrentTask(Task, NotThreadSafe); + +// Make all `Task` methods available on `CurrentTask`. +impl Deref for CurrentTask { + type Target = Task; + #[inline] + fn deref(&self) -> &Task { + &self.0 + } +} + +/// The type of process identifiers (PIDs). +pub type Pid = bindings::pid_t; + +/// The type of user identifiers (UIDs). +#[derive(Copy, Clone)] +pub struct Kuid { + kuid: bindings::kuid_t, +} + +impl Task { + /// Returns a raw pointer to the current task. + /// + /// It is up to the user to use the pointer correctly. + #[inline] + pub fn current_raw() -> *mut bindings::task_struct { + // SAFETY: Getting the current pointer is always safe. + unsafe { bindings::get_current() } + } + + /// Returns a task reference for the currently executing task/thread. + /// + /// The recommended way to get the current task/thread is to use the + /// [`current`] macro because it is safe. + /// + /// # Safety + /// + /// Callers must ensure that the returned object is only used to access a [`CurrentTask`] + /// within the task context that was active when this function was called. For more details, + /// see the invariants section for [`CurrentTask`]. + #[inline] + pub unsafe fn current() -> impl Deref { + struct TaskRef { + task: *const CurrentTask, + } + + impl Deref for TaskRef { + type Target = CurrentTask; + + fn deref(&self) -> &Self::Target { + // SAFETY: The returned reference borrows from this `TaskRef`, so it cannot outlive + // the `TaskRef`, which the caller of `Task::current()` has promised will not + // outlive the task/thread for which `self.task` is the `current` pointer. Thus, it + // is okay to return a `CurrentTask` reference here. + unsafe { &*self.task } + } + } + + TaskRef { + // CAST: The layout of `struct task_struct` and `CurrentTask` is identical. + task: Task::current_raw().cast(), + } + } + + /// Returns a raw pointer to the task. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::task_struct { + self.0.get() + } + + /// Returns the PID of the given task. + pub fn pid(&self) -> Pid { + // SAFETY: The pid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).pid) } + } + + /// Returns the TGID (Thread Group ID / Process ID) of the given task. + pub fn tgid(&self) -> Pid { + // SAFETY: The tgid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).tgid) } + } + + /// Returns the objective real UID of the given task. + #[inline] + pub fn uid(&self) -> Kuid { + // SAFETY: It's always safe to call `task_uid` on a valid task. + Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) + } + + /// Determines whether the given task has pending signals. + #[inline] + pub fn signal_pending(&self) -> bool { + // SAFETY: It's always safe to call `signal_pending` on a valid task. + unsafe { bindings::signal_pending(self.as_ptr()) != 0 } + } + + /// Returns task's pid namespace with elevated reference count + #[inline] + pub fn get_pid_ns(&self) -> Option> { + // SAFETY: By the type invariant, we know that `self.0` is valid. + let ptr = unsafe { bindings::task_get_pid_ns(self.as_ptr()) }; + if ptr.is_null() { + None + } else { + // SAFETY: `ptr` is valid by the safety requirements of this function. And we own a + // reference count via `task_get_pid_ns()`. + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::pid_namespace`. + Some(unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(ptr.cast::())) }) + } + } + + /// Returns the given task's pid in the provided pid namespace. + #[doc(alias = "task_tgid_nr_ns")] + #[inline] + pub fn tgid_nr_ns(&self, pidns: Option<&PidNamespace>) -> Pid { + let pidns = match pidns { + Some(pidns) => pidns.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: By the type invariant, we know that `self.0` is valid. We received a valid + // PidNamespace that we can use as a pointer or we received an empty PidNamespace and + // thus pass a null pointer. The underlying C function is safe to be used with NULL + // pointers. + unsafe { bindings::task_tgid_nr_ns(self.as_ptr(), pidns) } + } + + /// Wakes up the task. + #[inline] + pub fn wake_up(&self) { + // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task + // running. + unsafe { bindings::wake_up_process(self.as_ptr()) }; + } +} + +impl CurrentTask { + /// Access the address space of the current task. + /// + /// This function does not touch the refcount of the mm. + #[inline] + pub fn mm(&self) -> Option<&MmWithUser> { + // SAFETY: The `mm` field of `current` is not modified from other threads, so reading it is + // not a data race. + let mm = unsafe { (*self.as_ptr()).mm }; + + if mm.is_null() { + return None; + } + + // SAFETY: If `current->mm` is non-null, then it references a valid mm with a non-zero + // value of `mm_users`. Furthermore, the returned `&MmWithUser` borrows from this + // `CurrentTask`, so it cannot escape the scope in which the current pointer was obtained. + // + // This is safe even if `kthread_use_mm()`/`kthread_unuse_mm()` are used. There are two + // relevant cases: + // * If the `&CurrentTask` was created before `kthread_use_mm()`, then it cannot be + // accessed during the `kthread_use_mm()`/`kthread_unuse_mm()` scope due to the + // `NotThreadSafe` field of `CurrentTask`. + // * If the `&CurrentTask` was created within a `kthread_use_mm()`/`kthread_unuse_mm()` + // scope, then the `&CurrentTask` cannot escape that scope, so the returned `&MmWithUser` + // also cannot escape that scope. + // In either case, it's not possible to read `current->mm` and keep using it after the + // scope is ended with `kthread_unuse_mm()`. + Some(unsafe { MmWithUser::from_raw(mm) }) + } + + /// Access the pid namespace of the current task. + /// + /// This function does not touch the refcount of the namespace or use RCU protection. + /// + /// To access the pid namespace of another task, see [`Task::get_pid_ns`]. + #[doc(alias = "task_active_pid_ns")] + #[inline] + pub fn active_pid_ns(&self) -> Option<&PidNamespace> { + // SAFETY: It is safe to call `task_active_pid_ns` without RCU protection when calling it + // on the current task. + let active_ns = unsafe { bindings::task_active_pid_ns(self.as_ptr()) }; + + if active_ns.is_null() { + return None; + } + + // The lifetime of `PidNamespace` is bound to `Task` and `struct pid`. + // + // The `PidNamespace` of a `Task` doesn't ever change once the `Task` is alive. + // + // From system call context retrieving the `PidNamespace` for the current task is always + // safe and requires neither RCU locking nor a reference count to be held. Retrieving the + // `PidNamespace` after `release_task()` for current will return `NULL` but no codepath + // like that is exposed to Rust. + // + // SAFETY: If `current`'s pid ns is non-null, then it references a valid pid ns. + // Furthermore, the returned `&PidNamespace` borrows from this `CurrentTask`, so it cannot + // escape the scope in which the current pointer was obtained, e.g. it cannot live past a + // `release_task()` call. + Some(unsafe { PidNamespace::from_ptr(active_ns) }) + } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: The type invariants guarantee that `Task` is always refcounted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Task { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_task_struct(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } + } +} + +impl PartialEq for Task { + #[inline] + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.as_ptr(), other.as_ptr()) + } +} + +impl Eq for Task {} + +impl Kuid { + /// Get the current subjective effective UID. + #[inline] + pub fn current_euid() -> Kuid { + // SAFETY: Just an FFI call. + Self::from_raw(unsafe { bindings::current_euid() }) + } + + /// Create a `Kuid` given the raw C type. + #[inline] + pub fn from_raw(kuid: bindings::kuid_t) -> Self { + Self { kuid } + } + + /// Turn this kuid into the raw C type. + #[inline] + pub fn into_raw(self) -> bindings::kuid_t { + self.kuid + } + + /// Converts this kernel UID into a userspace UID. + /// + /// Uses the namespace of the current task. + #[inline] + pub fn into_uid_in_current_ns(self) -> bindings::uid_t { + // SAFETY: Just an FFI call. + unsafe { bindings::from_kuid(bindings::current_user_ns(), self.kuid) } + } +} + +impl PartialEq for Kuid { + #[inline] + fn eq(&self, other: &Kuid) -> bool { + // SAFETY: Just an FFI call. + unsafe { bindings::uid_eq(self.kuid, other.kuid) } + } +} + +impl Eq for Kuid {} + +/// Annotation for functions that can sleep. +/// +/// Equivalent to the C side [`might_sleep()`], this function serves as +/// a debugging aid and a potential scheduling point. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`might_sleep()`]: https://docs.kernel.org/driver-api/basics.html#c.might_sleep +#[track_caller] +#[inline] +pub fn might_sleep() { + #[cfg(CONFIG_DEBUG_ATOMIC_SLEEP)] + { + let loc = core::panic::Location::caller(); + let file = kernel::file_from_location(loc); + + // SAFETY: `file.as_ptr()` is valid for reading and guaranteed to be nul-terminated. + unsafe { crate::bindings::__might_sleep(file.as_char_ptr(), loc.line() as i32) } + } + + // SAFETY: Always safe to call. + unsafe { crate::bindings::might_resched() } +} diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs new file mode 100644 index 000000000..6c0a5e809 --- /dev/null +++ b/rust/kernel/time.rs @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Time related primitives. +//! +//! This module contains the kernel APIs related to time and timers that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! There are two types in this module: +//! +//! - The [`Instant`] type represents a specific point in time. +//! - The [`Delta`] type represents a span of time. +//! +//! Note that the C side uses `ktime_t` type to represent both. However, timestamp +//! and timedelta are different. To avoid confusion, we use two different types. +//! +//! A [`Instant`] object can be created by calling the [`Instant::now()`] function. +//! It represents a point in time at which the object was created. +//! By calling the [`Instant::elapsed()`] method, a [`Delta`] object representing +//! the elapsed time can be created. The [`Delta`] object can also be created +//! by subtracting two [`Instant`] objects. +//! +//! A [`Delta`] type supports methods to retrieve the duration in various units. +//! +//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). +//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). + +use core::marker::PhantomData; +use core::ops; + +pub mod delay; +pub mod hrtimer; + +/// The number of nanoseconds per microsecond. +pub const NSEC_PER_USEC: i64 = bindings::NSEC_PER_USEC as i64; + +/// The number of nanoseconds per millisecond. +pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; + +/// The number of nanoseconds per second. +pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64; + +/// The time unit of Linux kernel. One jiffy equals (1/HZ) second. +pub type Jiffies = crate::ffi::c_ulong; + +/// The millisecond time unit. +pub type Msecs = crate::ffi::c_uint; + +/// Converts milliseconds to jiffies. +#[inline] +pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { + // SAFETY: The `__msecs_to_jiffies` function is always safe to call no + // matter what the argument is. + unsafe { bindings::__msecs_to_jiffies(msecs) } +} + +/// Trait for clock sources. +/// +/// Selection of the clock source depends on the use case. In some cases the usage of a +/// particular clock is mandatory, e.g. in network protocols, filesystems. In other +/// cases the user of the clock has to decide which clock is best suited for the +/// purpose. In most scenarios clock [`Monotonic`] is the best choice as it +/// provides a accurate monotonic notion of time (leap second smearing ignored). +/// +/// # Safety +/// +/// Implementers must ensure that `ktime_get()` returns a value in the inclusive range +/// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to +/// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`). +pub unsafe trait ClockSource { + /// The kernel clock ID associated with this clock source. + /// + /// This constant corresponds to the C side `clockid_t` value. + const ID: bindings::clockid_t; + + /// Get the current time from the clock source. + /// + /// The function must return a value in the range `0..=KTIME_MAX`. + fn ktime_get() -> bindings::ktime_t; +} + +/// A monotonically increasing clock. +/// +/// A nonsettable system-wide clock that represents monotonic time since as +/// described by POSIX, "some unspecified point in the past". On Linux, that +/// point corresponds to the number of seconds that the system has been +/// running since it was booted. +/// +/// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the +/// CLOCK_REAL (e.g., if the system administrator manually changes the +/// clock), but is affected by frequency adjustments. This clock does not +/// count time that the system is suspended. +pub struct Monotonic; + +// SAFETY: The kernel's `ktime_get()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Monotonic { + const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get()` outside of NMI context. + unsafe { bindings::ktime_get() } + } +} + +/// A settable system-wide clock that measures real (i.e., wall-clock) time. +/// +/// Setting this clock requires appropriate privileges. This clock is +/// affected by discontinuous jumps in the system time (e.g., if the system +/// administrator manually changes the clock), and by frequency adjustments +/// performed by NTP and similar applications via adjtime(3), adjtimex(2), +/// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the +/// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time +/// (UTC) except that it ignores leap seconds; near a leap second it may be +/// adjusted by leap second smearing to stay roughly in sync with UTC. Leap +/// second smearing applies frequency adjustments to the clock to speed up +/// or slow down the clock to account for the leap second without +/// discontinuities in the clock. If leap second smearing is not applied, +/// the clock will experience discontinuity around leap second adjustment. +pub struct RealTime; + +// SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for RealTime { + const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_real()` outside of NMI context. + unsafe { bindings::ktime_get_real() } + } +} + +/// A monotonic that ticks while system is suspended. +/// +/// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC, +/// except that it also includes any time that the system is suspended. This +/// allows applications to get a suspend-aware monotonic clock without +/// having to deal with the complications of CLOCK_REALTIME, which may have +/// discontinuities if the time is changed using settimeofday(2) or similar. +pub struct BootTime; + +// SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for BootTime { + const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_boottime()` outside of NMI context. + unsafe { bindings::ktime_get_boottime() } + } +} + +/// International Atomic Time. +/// +/// A system-wide clock derived from wall-clock time but counting leap seconds. +/// +/// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is +/// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This +/// usually happens during boot and **should** not happen during normal operations. +/// However, if NTP or another application adjusts CLOCK_REALTIME by leap second +/// smearing, this clock will not be precise during leap second smearing. +/// +/// The acronym TAI refers to International Atomic Time. +pub struct Tai; + +// SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Tai { + const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_tai()` outside of NMI context. + unsafe { bindings::ktime_get_clocktai() } + } +} + +/// A specific point in time. +/// +/// # Invariants +/// +/// The `inner` value is in the range from 0 to `KTIME_MAX`. +#[repr(transparent)] +#[derive(PartialEq, PartialOrd, Eq, Ord)] +pub struct Instant { + inner: bindings::ktime_t, + _c: PhantomData, +} + +impl Clone for Instant { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Instant {} + +impl Instant { + /// Get the current time from the clock source. + #[inline] + pub fn now() -> Self { + // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range + // from 0 to `KTIME_MAX`. + Self { + inner: C::ktime_get(), + _c: PhantomData, + } + } + + /// Return the amount of time elapsed since the [`Instant`]. + #[inline] + pub fn elapsed(&self) -> Delta { + Self::now() - *self + } + + #[inline] + pub(crate) fn as_nanos(&self) -> i64 { + self.inner + } + + /// Create an [`Instant`] from a `ktime_t` without checking if it is non-negative. + /// + /// # Panics + /// + /// On debug builds, this function will panic if `ktime` is not in the range from 0 to + /// `KTIME_MAX`. + /// + /// # Safety + /// + /// The caller promises that `ktime` is in the range from 0 to `KTIME_MAX`. + #[inline] + pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self { + debug_assert!(ktime >= 0); + + // INVARIANT: Our safety contract ensures that `ktime` is in the range from 0 to + // `KTIME_MAX`. + Self { + inner: ktime, + _c: PhantomData, + } + } +} + +impl ops::Sub for Instant { + type Output = Delta; + + // By the type invariant, it never overflows. + #[inline] + fn sub(self, other: Instant) -> Delta { + Delta { + value: self.inner - other.inner, + } + } +} + +impl ops::Add for Instant { + type Output = Self; + + #[inline] + fn add(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner + rhs.value; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + +impl ops::Sub for Instant { + type Output = Self; + + #[inline] + fn sub(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner - rhs.value; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::Nsec {} + impl Sealed for super::Jiffy {} +} + +/// A trait for time units. +pub trait TimeUnit: private::Sealed { + /// The underlying representation of the time unit. + type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; +} + +/// A time unit of nanoseconds. +/// +/// A [`Delta`] stores its value as [`i64`] nanoseconds and can represent +/// any [`i64`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Nsec {} + +impl TimeUnit for Nsec { + type Repr = i64; +} + +/// A time unit of jiffies. +/// +/// A [`Delta`] stores its value as [`isize`] jiffies and can represent +/// any [`isize`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Jiffy {} + +impl TimeUnit for Jiffy { + type Repr = isize; +} + +/// A span of time. +/// +/// The span is stored in the unit given by the type parameter `U` (see +/// [`TimeUnit`]); its value has type `U::Repr`. `U` defaults to [`Nsec`], so a +/// plain [`Delta`] is a span in nanoseconds. The value can be negative, zero, or +/// positive. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub struct Delta { + value: U::Repr, +} + +impl Delta { + /// Create a new [`Delta`] from a number of jiffies. + #[inline] + pub const fn from_jiffies(jiffies: isize) -> Self { + Self { value: jiffies } + } + + /// Return the number of jiffies in the [`Delta`]. + #[inline] + pub const fn as_jiffies(self) -> isize { + self.value + } +} + +impl ops::Add for Delta { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> Self { + Self { + value: self.value + rhs.value, + } + } +} + +impl ops::AddAssign for Delta { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.value += rhs.value; + } +} + +impl ops::Sub for Delta { + type Output = Self; + + #[inline] + fn sub(self, rhs: Self) -> Self::Output { + Self { + value: self.value - rhs.value, + } + } +} + +impl ops::SubAssign for Delta { + #[inline] + fn sub_assign(&mut self, rhs: Self) { + self.value -= rhs.value; + } +} + +impl ops::Mul for Delta { + type Output = Self; + + #[inline] + fn mul(self, rhs: i64) -> Self::Output { + Self { + value: self.value * rhs, + } + } +} + +impl ops::MulAssign for Delta { + #[inline] + fn mul_assign(&mut self, rhs: i64) { + self.value *= rhs; + } +} + +impl ops::Div for Delta { + type Output = i64; + + #[inline] + fn div(self, rhs: Self) -> Self::Output { + #[cfg(CONFIG_64BIT)] + { + self.value / rhs.value + } + + #[cfg(not(CONFIG_64BIT))] + { + // SAFETY: This function is always safe to call regardless of the input values + unsafe { bindings::div64_s64(self.value, rhs.value) } + } + } +} + +impl Delta { + /// A span of time equal to zero. + pub const ZERO: Self = Self { value: 0 }; + + /// Create a new [`Delta`] from a number of nanoseconds. + #[inline] + pub const fn from_nanos(nanos: i64) -> Self { + Self { value: nanos } + } + + /// Create a new [`Delta`] from a number of microseconds. + /// + /// The `micros` can range from -9_223_372_036_854_775 to 9_223_372_036_854_775. + /// If `micros` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_micros(micros: i64) -> Self { + Self { + value: micros.saturating_mul(NSEC_PER_USEC), + } + } + + /// Create a new [`Delta`] from a number of milliseconds. + /// + /// The `millis` can range from -9_223_372_036_854 to 9_223_372_036_854. + /// If `millis` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_millis(millis: i64) -> Self { + Self { + value: millis.saturating_mul(NSEC_PER_MSEC), + } + } + + /// Create a new [`Delta`] from a number of seconds. + /// + /// The `secs` can range from -9_223_372_036 to 9_223_372_036. + /// If `secs` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_secs(secs: i64) -> Self { + Self { + value: secs.saturating_mul(NSEC_PER_SEC), + } + } + + /// Return `true` if the [`Delta`] spans no time. + #[inline] + pub fn is_zero(self) -> bool { + self.as_nanos() == 0 + } + + /// Return `true` if the [`Delta`] spans a negative amount of time. + #[inline] + pub fn is_negative(self) -> bool { + self.as_nanos() < 0 + } + + /// Return the number of nanoseconds in the [`Delta`]. + #[inline] + pub const fn as_nanos(self) -> i64 { + self.value + } + + /// Return the smallest number of microseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_USEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_us()` with any value. + unsafe { + bindings::ktime_to_us(n) + add + } + } + + /// Return the number of milliseconds in the [`Delta`]. + #[inline] + pub fn as_millis(self) -> i64 { + #[cfg(CONFIG_64BIT)] + { + self.as_nanos() / NSEC_PER_MSEC + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(self.as_nanos()) + } + } + + /// Return the smallest number of milliseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_millis_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_MSEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(n) + add + } + } + + /// Return `self % dividend` where `dividend` is in nanoseconds. + /// + /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is + /// limited to 32 bit dividends. + #[inline] + pub fn rem_nanos(self, dividend: i32) -> Self { + #[cfg(CONFIG_64BIT)] + { + Self { + value: self.as_nanos() % i64::from(dividend), + } + } + + #[cfg(not(CONFIG_64BIT))] + { + let mut rem = 0; + + // SAFETY: `rem` is in the stack, so we can always provide a valid pointer to it. + unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; + + Self { + value: i64::from(rem), + } + } + } +} diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs new file mode 100644 index 000000000..b5b1b4279 --- /dev/null +++ b/rust/kernel/time/delay.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Delay and sleep primitives. +//! +//! This module contains the kernel APIs related to delay and sleep that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h). + +use super::Delta; +use crate::prelude::*; + +/// Sleeps for a given duration at least. +/// +/// Equivalent to the C side [`fsleep()`], flexible sleep function, +/// which automatically chooses the best sleep method based on a duration. +/// +/// `delta` must be within `[0, i32::MAX]` microseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug +/// to call this function with an out-of-range value, in which case the function +/// will sleep for at least the maximum value in the range and may warn +/// in the future. +/// +/// The behavior above differs from the C side [`fsleep()`] for which out-of-range +/// values mean "infinite timeout" instead. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`fsleep()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.fsleep +pub fn fsleep(delta: Delta) { + // The maximum value is set to `i32::MAX` microseconds to prevent integer + // overflow inside fsleep, which could lead to unintentional infinite sleep. + const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64); + + let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) { + delta + } else { + // TODO: Add WARN_ONCE() when it's supported. + MAX_DELTA + }; + + // SAFETY: It is always safe to call `fsleep()` with any duration. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `fsleep()` sleeps for at least the provided duration, + // but that it may sleep for longer under some circumstances. + bindings::fsleep(delta.as_micros_ceil() as c_ulong) + } +} + +/// Inserts a delay based on microseconds with busy waiting. +/// +/// Equivalent to the C side [`udelay()`], which delays in microseconds. +/// +/// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug to +/// call this function with an out-of-range value. +/// +/// The behavior above differs from the C side [`udelay()`] for which out-of-range +/// values could lead to an overflow and unexpected behavior. +/// +/// [`udelay()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.udelay +pub fn udelay(delta: Delta) { + const MAX_UDELAY_DELTA: Delta = Delta::from_millis(bindings::MAX_UDELAY_MS as i64); + + debug_assert!(delta.as_nanos() >= 0); + debug_assert!(delta <= MAX_UDELAY_DELTA); + + let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) { + delta + } else { + MAX_UDELAY_DELTA + }; + + // SAFETY: It is always safe to call `udelay()` with any duration. + // Note that the kernel is compiled with `-fno-strict-overflow` + // so any out-of-range value could lead to unexpected behavior + // but won't lead to undefined behavior. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `udelay()` inserts a delay for at least + // the provided duration, but that it may delay for longer + // under some circumstances. + bindings::udelay(delta.as_micros_ceil() as c_ulong) + } +} diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs new file mode 100644 index 000000000..2d7f1131a --- /dev/null +++ b/rust/kernel/time/hrtimer.rs @@ -0,0 +1,1120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Intrusive high resolution timers. +//! +//! Allows running timer callbacks without doing allocations at the time of +//! starting the timer. For now, only one timer per type is allowed. +//! +//! # Vocabulary +//! +//! States: +//! +//! - Stopped: initialized but not started, or cancelled, or not restarted. +//! - Started: initialized and started or restarted. +//! - Running: executing the callback. +//! +//! Operations: +//! +//! * Start +//! * Cancel +//! * Restart +//! +//! Events: +//! +//! * Expire +//! +//! ## State Diagram +//! +//! ```text +//! Return NoRestart +//! +---------------------------------------------------------------------+ +//! | | +//! | | +//! | | +//! | Return Restart | +//! | +------------------------+ | +//! | | | | +//! | | | | +//! v v | | +//! +-----------------+ Start +------------------+ +--------+-----+--+ +//! | +---------------->| | | | +//! Init | | | | Expire | | +//! --------->| Stopped | | Started +---------->| Running | +//! | | Cancel | | | | +//! | |<----------------+ | | | +//! +-----------------+ +---------------+--+ +-----------------+ +//! ^ | +//! | | +//! +---------+ +//! Restart +//! ``` +//! +//! +//! A timer is initialized in the **stopped** state. A stopped timer can be +//! **started** by the `start` operation, with an **expiry** time. After the +//! `start` operation, the timer is in the **started** state. When the timer +//! **expires**, the timer enters the **running** state and the handler is +//! executed. After the handler has returned, the timer may enter the +//! **started* or **stopped** state, depending on the return value of the +//! handler. A timer in the **started** or **running** state may be **canceled** +//! by the `cancel` operation. A timer that is cancelled enters the **stopped** +//! state. +//! +//! A `cancel` or `restart` operation on a timer in the **running** state takes +//! effect after the handler has returned and the timer has transitioned +//! out of the **running** state. +//! +//! A `restart` operation on a timer in the **stopped** state is equivalent to a +//! `start` operation. +//! +//! When a type implements both `HrTimerPointer` and `Clone`, it is possible to +//! issue the `start` operation while the timer is in the **started** state. In +//! this case the `start` operation is equivalent to the `restart` operation. +//! +//! # Examples +//! +//! ## Using an intrusive timer living in a [`Box`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, +//! # HrTimerRestart, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct BoxIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! shared: Arc, +//! } +//! +//! impl BoxIntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for BoxIntrusiveHrTimer { +//! type Pointer<'a> = Pin>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for BoxIntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! let has_timer = Box::pin_init(BoxIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let shared = has_timer.shared.clone(); +//! let _handle = has_timer.start(Delta::from_micros(200)); +//! +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using an intrusive timer in an [`Arc`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, ArcBorrow, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct ArcIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl ArcIntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for ArcIntrusiveHrTimer { +//! type Pointer<'a> = Arc; +//! +//! fn run( +//! this: ArcBorrow<'_, Self>, +//! _ctx: HrTimerCallbackContext<'_, Self>, +//! ) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.flag.fetch_add(1, ordering::Full); +//! this.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for ArcIntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! let has_timer = Arc::pin_init(ArcIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let _handle = has_timer.clone().start(Delta::from_micros(200)); +//! +//! while has_timer.flag.load(ordering::Relaxed) != 5 { +//! has_timer.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_pin_init; +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a Self>; +//! +//! fn run(this: Pin<&Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! this.flag.store(1, ordering::Release); +//! this.cond.complete_all(); +//! +//! HrTimerRestart::NoRestart +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for IntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! stack_pin_init!( let has_timer = IntrusiveHrTimer::new() ); +//! has_timer.as_ref().start_scoped(Delta::from_micros(200), || { +//! while has_timer.flag.load(ordering::Relaxed) != 1 { +//! has_timer.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Flag raised\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a mutable stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_try_pin_init; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! shared: Arc, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a mut Self>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for IntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! stack_try_pin_init!( let has_timer =? IntrusiveHrTimer::new() ); +//! let shared = has_timer.shared.clone(); +//! +//! has_timer.as_mut().start_scoped(Delta::from_micros(200), || { +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! [`Arc`]: kernel::sync::Arc + +use super::{ClockSource, Delta, Instant}; +use crate::{prelude::*, types::Opaque}; +use core::{marker::PhantomData, ptr::NonNull}; +use pin_init::PinInit; + +/// A type-alias to refer to the [`Instant`] for a given `T` from [`HrTimer`]. +/// +/// Where `C` is the [`ClockSource`] of the [`HrTimer`]. +pub type HrTimerInstant = Instant<<>::TimerMode as HrTimerMode>::Clock>; + +/// A timer backed by a C `struct hrtimer`. +/// +/// # Invariants +/// +/// * `self.timer` is initialized by `bindings::hrtimer_setup`. +#[pin_data] +#[repr(C)] +pub struct HrTimer { + #[pin] + timer: Opaque, + _t: PhantomData, +} + +// SAFETY: Ownership of an `HrTimer` can be moved to other threads and +// used/dropped from there. +unsafe impl Send for HrTimer {} + +// SAFETY: Timer operations are locked on the C side, so it is safe to operate +// on a timer from multiple threads. +unsafe impl Sync for HrTimer {} + +impl HrTimer { + /// Return an initializer for a new timer instance. + pub fn new() -> impl PinInit + where + T: HrTimerCallback, + T: HasHrTimer, + { + pin_init!(Self { + // INVARIANT: We initialize `timer` with `hrtimer_setup` below. + timer <- Opaque::ffi_init(move |place: *mut bindings::hrtimer| { + // SAFETY: By design of `pin_init!`, `place` is a pointer to a + // live allocation. hrtimer_setup will initialize `place` and + // does not require `place` to be initialized prior to the call. + unsafe { + bindings::hrtimer_setup( + place, + Some(T::Pointer::run), + <>::TimerMode as HrTimerMode>::Clock::ID, + >::TimerMode::C_MODE, + ); + } + }), + _t: PhantomData, + }) + } + + /// Get a pointer to the contained `bindings::hrtimer`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a live allocation of at least the size of `Self`. + unsafe fn raw_get(this: *const Self) -> *mut bindings::hrtimer { + // SAFETY: The field projection to `timer` does not go out of bounds, + // because the caller of this function promises that `this` points to an + // allocation of at least the size of `Self`. + unsafe { Opaque::cast_into(core::ptr::addr_of!((*this).timer)) } + } + + /// Cancel an initialized and potentially running timer. + /// + /// If the timer handler is running, this function will block until the + /// handler returns. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + /// + /// Users of the `HrTimer` API would not usually call this method directly. + /// Instead they would use the safe [`HrTimerHandle::cancel`] on the handle + /// returned when the timer was started. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a valid `Self`. + pub(crate) unsafe fn raw_cancel(this: *const Self) -> bool { + // SAFETY: `this` points to an allocation of at least `HrTimer` size. + let c_timer_ptr = unsafe { HrTimer::raw_get(this) }; + + // If the handler is running, this will wait for the handler to return + // before returning. + // SAFETY: `c_timer_ptr` is initialized and valid. Synchronization is + // handled on the C side. + unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 } + } + + /// Forward the timer expiry for a given timer pointer. + /// + /// # Safety + /// + /// - `self_ptr` must point to a valid `Self`. + /// - The caller must either have exclusive access to the data pointed at by `self_ptr`, or be + /// within the context of the timer callback. + #[inline] + unsafe fn raw_forward(self_ptr: *mut Self, now: HrTimerInstant, interval: Delta) -> u64 + where + T: HasHrTimer, + { + // SAFETY: + // * The C API requirements for this function are fulfilled by our safety contract. + // * `self_ptr` is guaranteed to point to a valid `Self` via our safety contract + unsafe { + bindings::hrtimer_forward(Self::raw_get(self_ptr), now.as_nanos(), interval.as_nanos()) + } + } + + /// Conditionally forward the timer. + /// + /// If the timer expires after `now`, this function does nothing and returns 0. If the timer + /// expired at or before `now`, this function forwards the timer by `interval` until the timer + /// expires after `now` and then returns the number of times the timer was forwarded by + /// `interval`. + /// + /// This function is mainly useful for timer types which can provide exclusive access to the + /// timer when the timer is not running. For forwarding the timer from within the timer callback + /// context, see [`HrTimerCallbackContext::forward()`]. + /// + /// Returns the number of overruns that occurred as a result of the timer expiry change. + pub fn forward(self: Pin<&mut Self>, now: HrTimerInstant, interval: Delta) -> u64 + where + T: HasHrTimer, + { + // SAFETY: `raw_forward` does not move `Self` + let this = unsafe { self.get_unchecked_mut() }; + + // SAFETY: By existence of `Pin<&mut Self>`, the pointer passed to `raw_forward` points to a + // valid `Self` that we have exclusive access to. + unsafe { Self::raw_forward(this, now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`forward()`](Self::forward) that uses an interval after the current + /// time of the base clock for the [`HrTimer`]. + pub fn forward_now(self: Pin<&mut Self>, interval: Delta) -> u64 + where + T: HasHrTimer, + { + self.forward(HrTimerInstant::::now(), interval) + } + + /// Return the time expiry for this [`HrTimer`]. + /// + /// This value should only be used as a snapshot, as the actual expiry time could change after + /// this function is called. + pub fn expires(&self) -> HrTimerInstant + where + T: HasHrTimer, + { + // SAFETY: `self` is an immutable reference and thus always points to a valid `HrTimer`. + let c_timer_ptr = unsafe { HrTimer::raw_get(self) }; + + // SAFETY: + // - Timers cannot have negative ktime_t values as their expiration time. + // - There's no actual locking here, a racy read is fine and expected + unsafe { + Instant::from_ktime( + // This `read_volatile` is intended to correspond to a READ_ONCE call. + // FIXME(read_once): Replace with `read_once` when available on the Rust side. + core::ptr::read_volatile(&raw const ((*c_timer_ptr).node.expires)), + ) + } + } +} + +/// Implemented by pointer types that point to structs that contain a [`HrTimer`]. +/// +/// `Self` must be [`Sync`] because it is passed to timer callbacks in another +/// thread of execution (hard or soft interrupt context). +/// +/// Starting a timer returns a [`HrTimerHandle`] that can be used to manipulate +/// the timer. Note that it is OK to call the start function repeatedly, and +/// that more than one [`HrTimerHandle`] associated with a [`HrTimerPointer`] may +/// exist. A timer can be manipulated through any of the handles, and a handle +/// may represent a cancelled timer. +pub trait HrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// A handle representing a started or restarted timer. + /// + /// If the timer is running or if the timer callback is executing when the + /// handle is dropped, the drop method of [`HrTimerHandle`] should not return + /// until the timer is stopped and the callback has completed. + /// + /// Note: When implementing this trait, consider that it is not unsafe to + /// leak the handle. + type TimerHandle: HrTimerHandle; + + /// Start the timer with expiry after `expires` time units. If the timer was + /// already running, it is restarted with the new expiry time. + fn start(self, expires: ::Expires) -> Self::TimerHandle; +} + +/// Unsafe version of [`HrTimerPointer`] for situations where leaking the +/// [`HrTimerHandle`] returned by `start` would be unsound. This is the case for +/// stack allocated timers. +/// +/// Typical implementers are pinned references such as [`Pin<&T>`]. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that instances of types implementing +/// [`UnsafeHrTimerPointer`] outlives any associated [`HrTimerPointer::TimerHandle`] +/// instances. +pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// A handle representing a running timer. + /// + /// # Safety + /// + /// If the timer is running, or if the timer callback is executing when the + /// handle is dropped, the drop method of [`Self::TimerHandle`] must not return + /// until the timer is stopped and the callback has completed. + type TimerHandle: HrTimerHandle; + + /// Start the timer after `expires` time units. If the timer was already + /// running, it is restarted at the new expiry time. + /// + /// # Safety + /// + /// Caller promises keep the timer structure alive until the timer is dead. + /// Caller can ensure this by not leaking the returned [`Self::TimerHandle`]. + unsafe fn start(self, expires: ::Expires) -> Self::TimerHandle; +} + +/// A trait for stack allocated timers. +/// +/// # Safety +/// +/// Implementers must ensure that `start_scoped` does not return until the +/// timer is dead and the timer handler is not running. +pub unsafe trait ScopedHrTimerPointer { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// Start the timer to run after `expires` time units and immediately + /// after call `f`. When `f` returns, the timer is cancelled. + fn start_scoped(self, expires: ::Expires, f: F) -> T + where + F: FnOnce() -> T; +} + +// SAFETY: By the safety requirement of [`UnsafeHrTimerPointer`], dropping the +// handle returned by [`UnsafeHrTimerPointer::start`] ensures that the timer is +// killed. +unsafe impl ScopedHrTimerPointer for T +where + T: UnsafeHrTimerPointer, +{ + type TimerMode = T::TimerMode; + + fn start_scoped( + self, + expires: <::TimerMode as HrTimerMode>::Expires, + f: F, + ) -> U + where + F: FnOnce() -> U, + { + // SAFETY: We drop the timer handle below before returning. + let handle = unsafe { UnsafeHrTimerPointer::start(self, expires) }; + let t = f(); + drop(handle); + t + } +} + +/// Implemented by [`HrTimerPointer`] implementers to give the C timer callback a +/// function to call. +// This is split from `HrTimerPointer` to make it easier to specify trait bounds. +pub trait RawHrTimerCallback { + /// Type of the parameter passed to [`HrTimerCallback::run`]. It may be + /// [`Self`], or a pointer type derived from [`Self`]. + type CallbackTarget<'a>; + + /// Callback to be called from C when timer fires. + /// + /// # Safety + /// + /// Only to be called by C code in the `hrtimer` subsystem. `this` must point + /// to the `bindings::hrtimer` structure that was used to start the timer. + unsafe extern "C" fn run(this: *mut bindings::hrtimer) -> bindings::hrtimer_restart; +} + +/// Implemented by structs that can be the target of a timer callback. +pub trait HrTimerCallback { + /// The type whose [`RawHrTimerCallback::run`] method will be invoked when + /// the timer expires. + type Pointer<'a>: RawHrTimerCallback; + + /// Called by the timer logic when the timer fires. + fn run( + this: as RawHrTimerCallback>::CallbackTarget<'_>, + ctx: HrTimerCallbackContext<'_, Self>, + ) -> HrTimerRestart + where + Self: Sized, + Self: HasHrTimer; +} + +/// A handle representing a potentially running timer. +/// +/// More than one handle representing the same timer might exist. +/// +/// # Safety +/// +/// When dropped, the timer represented by this handle must be cancelled, if it +/// is running. If the timer handler is running when the handle is dropped, the +/// drop method must wait for the handler to return before returning. +/// +/// Note: One way to satisfy the safety requirement is to call `Self::cancel` in +/// the drop implementation for `Self.` +pub unsafe trait HrTimerHandle { + /// Cancel the timer. If the timer is in the running state, block till the + /// handler has returned. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + /// + /// Returns `true` if the timer was running. + fn cancel(&mut self) -> bool; +} + +/// Implemented by structs that contain timer nodes. +/// +/// Clients of the timer API would usually safely implement this trait by using +/// the [`crate::impl_has_hr_timer`] macro. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that the implementer has a +/// [`HrTimer`] field and that all trait methods are implemented according to +/// their documentation. All the methods of this trait must operate on the same +/// field. +pub unsafe trait HasHrTimer { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// Return a pointer to the [`HrTimer`] within `Self`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn raw_get_timer(this: *const Self) -> *const HrTimer; + + /// Return a pointer to the struct that is containing the [`HrTimer`] pointed + /// to by `ptr`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `ptr` must point to a [`HrTimer`] field in a struct of type `Self`. + unsafe fn timer_container_of(ptr: *mut HrTimer) -> *mut Self + where + Self: Sized; + + /// Get pointer to the contained `bindings::hrtimer` struct. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn c_timer_ptr(this: *const Self) -> *const bindings::hrtimer { + // SAFETY: `this` is a valid pointer to a `Self`. + let timer_ptr = unsafe { Self::raw_get_timer(this) }; + + // SAFETY: timer_ptr points to an allocation of at least `HrTimer` size. + unsafe { HrTimer::raw_get(timer_ptr) } + } + + /// Start the timer contained in the `Self` pointed to by `self_ptr`. If + /// it is already running it is removed and inserted. + /// + /// # Safety + /// + /// - `this` must point to a valid `Self`. + /// - Caller must ensure that the pointee of `this` lives until the timer + /// fires or is canceled. + unsafe fn start(this: *const Self, expires: ::Expires) { + // SAFETY: By function safety requirement, `this` is a valid `Self`. + unsafe { + bindings::hrtimer_start_range_ns( + Self::c_timer_ptr(this).cast_mut(), + expires.as_nanos(), + 0, + ::C_MODE, + ); + } + } +} + +/// Restart policy for timers. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum HrTimerRestart { + /// Timer should not be restarted. + NoRestart = bindings::hrtimer_restart_HRTIMER_NORESTART, + /// Timer should be restarted. + Restart = bindings::hrtimer_restart_HRTIMER_RESTART, +} + +impl HrTimerRestart { + fn into_c(self) -> bindings::hrtimer_restart { + self as bindings::hrtimer_restart + } +} + +/// Time representations that can be used as expiration values in [`HrTimer`]. +pub trait HrTimerExpires { + /// Converts the expiration time into a nanosecond representation. + /// + /// This value corresponds to a raw ktime_t value, suitable for passing to kernel + /// timer functions. The interpretation (absolute vs relative) depends on the + /// associated [HrTimerMode] in use. + fn as_nanos(&self) -> i64; +} + +impl HrTimerExpires for Instant { + #[inline] + fn as_nanos(&self) -> i64 { + Instant::::as_nanos(self) + } +} + +impl HrTimerExpires for Delta { + #[inline] + fn as_nanos(&self) -> i64 { + Delta::as_nanos(*self) + } +} + +mod private { + use crate::time::ClockSource; + + pub trait Sealed {} + + impl Sealed for super::AbsoluteMode {} + impl Sealed for super::RelativeMode {} + impl Sealed for super::AbsolutePinnedMode {} + impl Sealed for super::RelativePinnedMode {} + impl Sealed for super::AbsoluteSoftMode {} + impl Sealed for super::RelativeSoftMode {} + impl Sealed for super::AbsolutePinnedSoftMode {} + impl Sealed for super::RelativePinnedSoftMode {} + impl Sealed for super::AbsoluteHardMode {} + impl Sealed for super::RelativeHardMode {} + impl Sealed for super::AbsolutePinnedHardMode {} + impl Sealed for super::RelativePinnedHardMode {} +} + +/// Operational mode of [`HrTimer`]. +pub trait HrTimerMode: private::Sealed { + /// The C representation of hrtimer mode. + const C_MODE: bindings::hrtimer_mode; + + /// Type representing the clock source. + type Clock: ClockSource; + + /// Type representing the expiration specification (absolute or relative time). + type Expires: HrTimerExpires; +} + +/// Timer that expires at a fixed point in time. +pub struct AbsoluteMode(PhantomData); + +impl HrTimerMode for AbsoluteMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS; + + type Clock = C; + type Expires = Instant; +} + +/// Timer that expires after a delay from now. +pub struct RelativeMode(PhantomData); + +impl HrTimerMode for RelativeMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration time, pinned to its current CPU. +pub struct AbsolutePinnedMode(PhantomData); +impl HrTimerMode for AbsolutePinnedMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration time, pinned to its current CPU. +pub struct RelativePinnedMode(PhantomData); +impl HrTimerMode for RelativePinnedMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in soft irq context. +pub struct AbsoluteSoftMode(PhantomData); +impl HrTimerMode for AbsoluteSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, handled in soft irq context. +pub struct RelativeSoftMode(PhantomData); +impl HrTimerMode for RelativeSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct AbsolutePinnedSoftMode(PhantomData); +impl HrTimerMode for AbsolutePinnedSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct RelativePinnedSoftMode(PhantomData); +impl HrTimerMode for RelativePinnedSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in hard irq context. +pub struct AbsoluteHardMode(PhantomData); +impl HrTimerMode for AbsoluteHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, handled in hard irq context. +pub struct RelativeHardMode(PhantomData); +impl HrTimerMode for RelativeHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in hard irq context. +pub struct AbsolutePinnedHardMode(PhantomData); +impl HrTimerMode for AbsolutePinnedHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, pinned to CPU and handled in hard irq context. +pub struct RelativePinnedHardMode(PhantomData); +impl HrTimerMode for RelativePinnedHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD; + + type Clock = C; + type Expires = Delta; +} + +/// Privileged smart-pointer for a [`HrTimer`] callback context. +/// +/// Many [`HrTimer`] methods can only be called in two situations: +/// +/// * When the caller has exclusive access to the `HrTimer` and the `HrTimer` is guaranteed not to +/// be running. +/// * From within the context of an `HrTimer`'s callback method. +/// +/// This type provides access to said methods from within a timer callback context. +/// +/// # Invariants +/// +/// * The existence of this type means the caller is currently within the callback for an +/// [`HrTimer`]. +/// * `self.0` always points to a live instance of [`HrTimer`]. +pub struct HrTimerCallbackContext<'a, T: HasHrTimer>(NonNull>, PhantomData<&'a ()>); + +impl<'a, T: HasHrTimer> HrTimerCallbackContext<'a, T> { + /// Create a new [`HrTimerCallbackContext`]. + /// + /// # Safety + /// + /// This function relies on the caller being within the context of a timer callback, so it must + /// not be used anywhere except for within implementations of [`RawHrTimerCallback::run`]. The + /// caller promises that `timer` points to a valid initialized instance of + /// [`bindings::hrtimer`]. + /// + /// The returned `Self` must not outlive the function context of [`RawHrTimerCallback::run`] + /// where this function is called. + pub(crate) unsafe fn from_raw(timer: *mut HrTimer) -> Self { + // SAFETY: The caller guarantees `timer` is a valid pointer to an initialized + // `bindings::hrtimer` + // INVARIANT: Our safety contract ensures that we're within the context of a timer callback + // and that `timer` points to a live instance of `HrTimer`. + Self(unsafe { NonNull::new_unchecked(timer) }, PhantomData) + } + + /// Conditionally forward the timer. + /// + /// This function is identical to [`HrTimer::forward()`] except that it may only be used from + /// within the context of a [`HrTimer`] callback. + pub fn forward(&mut self, now: HrTimerInstant, interval: Delta) -> u64 { + // SAFETY: + // - We are guaranteed to be within the context of a timer callback by our type invariants + // - By our type invariants, `self.0` always points to a valid `HrTimer` + unsafe { HrTimer::::raw_forward(self.0.as_ptr(), now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`HrTimerCallbackContext::forward()`] that uses an interval after the + /// current time of the base clock for the [`HrTimer`]. + pub fn forward_now(&mut self, duration: Delta) -> u64 { + self.forward(HrTimerInstant::::now(), duration) + } +} + +/// Use to implement the [`HasHrTimer`] trait. +/// +/// See [`module`] documentation for an example. +/// +/// [`module`]: crate::time::hrtimer +#[macro_export] +macro_rules! impl_has_hr_timer { + ( + impl$({$($generics:tt)*})? + HasHrTimer<$timer_type:ty> + for $self:ty + { + mode : $mode:ty, + field : self.$field:ident $(,)? + } + $($rest:tt)* + ) => { + // SAFETY: This implementation of `raw_get_timer` only compiles if the + // field has the right type. + unsafe impl$(<$($generics)*>)? $crate::time::hrtimer::HasHrTimer<$timer_type> for $self { + type TimerMode = $mode; + + #[inline] + unsafe fn raw_get_timer( + this: *const Self, + ) -> *const $crate::time::hrtimer::HrTimer<$timer_type> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { ::core::ptr::addr_of!((*this).$field) } + } + + #[inline] + unsafe fn timer_container_of( + ptr: *mut $crate::time::hrtimer::HrTimer<$timer_type>, + ) -> *mut Self { + // SAFETY: As per the safety requirement of this function, `ptr` + // is pointing inside a `$timer_type`. + unsafe { ::kernel::container_of!(ptr, $timer_type, $field) } + } + } + } +} + +mod arc; +pub use arc::ArcHrTimerHandle; +mod pin; +pub use pin::PinHrTimerHandle; +mod pin_mut; +pub use pin_mut::PinMutHrTimerHandle; +// `box` is a reserved keyword, so prefix with `t` for timer +mod tbox; +pub use tbox::BoxHrTimerHandle; diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs new file mode 100644 index 000000000..7be82bcb3 --- /dev/null +++ b/rust/kernel/time/hrtimer/arc.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::sync::Arc; +use crate::sync::ArcBorrow; + +/// A handle for an `Arc>` returned by a call to +/// [`HrTimerPointer::start`]. +pub struct ArcHrTimerHandle +where + T: HasHrTimer, +{ + pub(crate) inner: Arc, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl HrTimerHandle for ArcHrTimerHandle +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr = Arc::as_ptr(&self.inner); + + // SAFETY: As we obtained `self_ptr` from a valid reference above, it + // must point to a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl Drop for ArcHrTimerHandle +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +impl HrTimerPointer for Arc +where + T: 'static, + T: Send + Sync, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = ArcHrTimerHandle; + + fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> ArcHrTimerHandle { + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(Arc::as_ptr(&self), expires) }; + ArcHrTimerHandle { inner: self } + } +} + +impl RawHrTimerCallback for Arc +where + T: 'static, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Self>, +{ + type CallbackTarget<'a> = ArcBorrow<'a, T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - `data_ptr` is derived form the pointer to the `T` that was used to + // queue the timer. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `ArcHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the `ArcBorrow` created below. + // - We own one refcount in the `ArcTimerHandle` associated with this + // timer, so it is not possible to get a `UniqueArc` to this + // allocation from other `Arc` clones. + let receiver = unsafe { ArcBorrow::from_raw(data_ptr) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs new file mode 100644 index 000000000..4d39ef781 --- /dev/null +++ b/rust/kernel/time/hrtimer/pin.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::RawHrTimerCallback; +use super::UnsafeHrTimerPointer; +use core::pin::Pin; + +/// A handle for a `Pin<&HasHrTimer>`. When the handle exists, the timer might be +/// running. +pub struct PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + pub(crate) inner: Pin<&'a T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr: *const T = self.inner.get_ref(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a `PinHrTimerHandle`, +// so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a T> +where + T: Send + Sync, + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = PinHrTimerHandle<'a, T>; + + unsafe fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // Cast to pointer + let self_ptr: *const T = self.get_ref(); + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr, expires) }; + + PinHrTimerHandle { inner: self } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a T> +where + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr`, thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &*receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs new file mode 100644 index 000000000..9d9447d4d --- /dev/null +++ b/rust/kernel/time/hrtimer/pin_mut.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::{ + HasHrTimer, HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerHandle, HrTimerMode, + RawHrTimerCallback, UnsafeHrTimerPointer, +}; +use core::{marker::PhantomData, pin::Pin, ptr::NonNull}; + +/// A handle for a `Pin<&mut HasHrTimer>`. When the handle exists, the timer might +/// be running. +pub struct PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + pub(crate) inner: NonNull, + _p: PhantomData<&'a mut T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr = self.inner.as_ptr(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a +// `PinMutHrTimerHandle`, so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a mut T> +where + T: Send + Sync, + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = PinMutHrTimerHandle<'a, T>; + + unsafe fn start( + mut self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // SAFETY: + // - We promise not to move out of `self`. We only pass `self` + // back to the caller as a `Pin<&mut self>`. + // - The return value of `get_unchecked_mut` is guaranteed not to be null. + let self_ptr = unsafe { NonNull::new_unchecked(self.as_mut().get_unchecked_mut()) }; + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr.as_ptr(), expires) }; + + PinMutHrTimerHandle { + inner: self_ptr, + _p: PhantomData, + } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a mut T> +where + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinMutHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr` mutably thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &mut *receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs new file mode 100644 index 000000000..aa1ee31a7 --- /dev/null +++ b/rust/kernel/time/hrtimer/tbox.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::prelude::*; +use core::ptr::NonNull; + +/// A handle for a [`Box>`] returned by a call to +/// [`HrTimerPointer::start`]. +/// +/// # Invariants +/// +/// - `self.inner` comes from a `Box::into_raw` call. +pub struct BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + pub(crate) inner: NonNull, + _p: core::marker::PhantomData, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl HrTimerHandle for BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + fn cancel(&mut self) -> bool { + // SAFETY: As we obtained `self.inner` from a valid reference when we + // created `self`, it must point to a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self.inner.as_ptr()) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl Drop for BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + fn drop(&mut self) { + self.cancel(); + // SAFETY: By type invariant, `self.inner` came from a `Box::into_raw` + // call. + drop(unsafe { Box::::from_raw(self.inner.as_ptr()) }) + } +} + +impl HrTimerPointer for Pin> +where + T: 'static, + T: Send + Sync, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Pin>>, + A: crate::alloc::Allocator, +{ + type TimerMode = >::TimerMode; + type TimerHandle = BoxHrTimerHandle; + + fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // SAFETY: + // - We will not move out of this box during timer callback (we pass an + // immutable reference to the callback). + // - `Box::into_raw` is guaranteed to return a valid pointer. + let inner = + unsafe { NonNull::new_unchecked(Box::into_raw(Pin::into_inner_unchecked(self))) }; + + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(inner.as_ptr(), expires) }; + + // INVARIANT: `inner` came from `Box::into_raw` above. + BoxHrTimerHandle { + inner, + _p: core::marker::PhantomData, + } + } +} + +impl RawHrTimerCallback for Pin> +where + T: 'static, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Pin>>, + A: crate::alloc::Allocator, +{ + type CallbackTarget<'a> = Pin<&'a mut T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `BoxHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle owns the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the reference created below. + // - As `data_ptr` comes from a `Pin>`, only pinned references to + // `data_ptr` exist. + let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(data_mut_ref, context).into_c() + } +} diff --git a/rust/kernel/tracepoint.rs b/rust/kernel/tracepoint.rs new file mode 100644 index 000000000..c6e80aa99 --- /dev/null +++ b/rust/kernel/tracepoint.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for tracepoints. + +/// Declare the Rust entry point for a tracepoint. +/// +/// This macro generates an unsafe function that calls into C, and its safety requirements will be +/// whatever the relevant C code requires. To document these safety requirements, you may add +/// doc-comments when invoking the macro. +#[macro_export] +macro_rules! declare_trace { + ($($(#[$attr:meta])* $pub:vis unsafe fn $name:ident($($argname:ident : $argtyp:ty),* $(,)?);)*) => {$( + $( #[$attr] )* + #[inline(always)] + $pub unsafe fn $name($($argname : $argtyp),*) { + #[cfg(CONFIG_TRACEPOINTS)] + { + // SAFETY: It's always okay to query the static key for a tracepoint. + let should_trace = unsafe { + $crate::macros::paste! { + $crate::jump_label::static_branch_unlikely!( + $crate::bindings::[< __tracepoint_ $name >], + $crate::bindings::tracepoint, + key + ) + } + }; + + if should_trace { + $crate::macros::paste! { + // SAFETY: The caller guarantees that it is okay to call this tracepoint. + unsafe { $crate::bindings::[< rust_do_trace_ $name >]($($argname),*) }; + } + } + } + + #[cfg(not(CONFIG_TRACEPOINTS))] + { + // If tracepoints are disabled, insert a trivial use of each argument + // to avoid unused argument warnings. + $( let _unused = $argname; )* + } + } + )*} +} + +pub use declare_trace; diff --git a/rust/kernel/transmute.rs b/rust/kernel/transmute.rs new file mode 100644 index 000000000..654b5ede2 --- /dev/null +++ b/rust/kernel/transmute.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Traits for transmuting types. + +use core::mem::size_of; + +/// Types for which any bit pattern is valid. +/// +/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so +/// reading arbitrary bytes into something that contains a `bool` is not okay. +/// +/// It's okay for the type to have padding, as initializing those bytes has no effect. +/// +/// # Examples +/// +/// ``` +/// use kernel::transmute::FromBytes; +/// +/// # fn test() -> Option<()> { +/// let raw = [1, 2, 3, 4]; +/// +/// let result = u32::from_bytes(&raw)?; +/// +/// #[cfg(target_endian = "little")] +/// assert_eq!(*result, 0x4030201); +/// +/// #[cfg(target_endian = "big")] +/// assert_eq!(*result, 0x1020304); +/// +/// # Some(()) } +/// # test().ok_or(EINVAL)?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Safety +/// +/// All bit-patterns must be valid for this type. This type must not have interior mutability. +pub unsafe trait FromBytes { + /// Converts a slice of bytes to a reference to `Self`. + /// + /// Succeeds if the reference is properly aligned, and the size of `bytes` is equal to that of + /// `T` and different from zero. + /// + /// Otherwise, returns [`None`]. + fn from_bytes(bytes: &[u8]) -> Option<&Self> + where + Self: Sized, + { + let slice_ptr = bytes.as_ptr().cast::(); + let size = size_of::(); + + if bytes.len() == size && slice_ptr.is_aligned() { + // SAFETY: Size and alignment were just checked. + unsafe { Some(&*slice_ptr) } + } else { + None + } + } + + /// Converts the beginning of `bytes` to a reference to `Self`. + /// + /// This method is similar to [`Self::from_bytes`], with the difference that `bytes` does not + /// need to be the same size of `Self` - the appropriate portion is cut from the beginning of + /// `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_prefix(bytes: &[u8]) -> Option<(&Self, &[u8])> + where + Self: Sized, + { + let (prefix, remainder) = bytes.split_at_checked(size_of::())?; + + Self::from_bytes(prefix).map(|s| (s, remainder)) + } + + /// Converts a mutable slice of bytes to a reference to `Self`. + /// + /// Succeeds if the reference is properly aligned, and the size of `bytes` is equal to that of + /// `T` and different from zero. + /// + /// Otherwise, returns [`None`]. + fn from_bytes_mut(bytes: &mut [u8]) -> Option<&mut Self> + where + Self: AsBytes + Sized, + { + let slice_ptr = bytes.as_mut_ptr().cast::(); + let size = size_of::(); + + if bytes.len() == size && slice_ptr.is_aligned() { + // SAFETY: Size and alignment were just checked. + unsafe { Some(&mut *slice_ptr) } + } else { + None + } + } + + /// Converts the beginning of `bytes` to a mutable reference to `Self`. + /// + /// This method is similar to [`Self::from_bytes_mut`], with the difference that `bytes` does + /// not need to be the same size of `Self` - the appropriate portion is cut from the beginning + /// of `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_mut_prefix(bytes: &mut [u8]) -> Option<(&mut Self, &mut [u8])> + where + Self: AsBytes + Sized, + { + let (prefix, remainder) = bytes.split_at_mut_checked(size_of::())?; + + Self::from_bytes_mut(prefix).map(|s| (s, remainder)) + } + + /// Creates an owned instance of `Self` by copying `bytes`. + /// + /// Unlike [`FromBytes::from_bytes`], which requires aligned input, this method can be used on + /// non-aligned data at the cost of a copy. + fn from_bytes_copy(bytes: &[u8]) -> Option + where + Self: Sized, + { + if bytes.len() == size_of::() { + // SAFETY: we just verified that `bytes` has the same size as `Self`, and per the + // invariants of `FromBytes`, any byte sequence of the correct length is a valid value + // for `Self`. + Some(unsafe { core::ptr::read_unaligned(bytes.as_ptr().cast::()) }) + } else { + None + } + } + + /// Creates an owned instance of `Self` from the beginning of `bytes`. + /// + /// This method is similar to [`Self::from_bytes_copy`], with the difference that `bytes` does + /// not need to be the same size of `Self` - the appropriate portion is cut from the beginning + /// of `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_copy_prefix(bytes: &[u8]) -> Option<(Self, &[u8])> + where + Self: Sized, + { + let (prefix, remainder) = bytes.split_at_checked(size_of::())?; + + Self::from_bytes_copy(prefix).map(|s| (s, remainder)) + } +} + +macro_rules! impl_frombytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? FromBytes for $t {})* + }; +} + +impl_frombytes! { + // SAFETY: Inhabited ZSTs only have one possible bit pattern, and these two have no invariant. + (), + {} core::marker::PhantomData, + + // SAFETY: All bit patterns are acceptable values of the types below. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + + // SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit + // patterns are also acceptable for arrays of that type. + {} [T], + {} [T; N], +} + +/// Types that can be viewed as an immutable slice of initialized bytes. +/// +/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This +/// means that it should not have any padding, as padding bytes are uninitialized. Reading +/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive +/// information on the stack to userspace. +/// +/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered +/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so +/// this is a correctness requirement, but not a safety requirement. +/// +/// # Safety +/// +/// Values of this type may not contain any uninitialized bytes. This type must not have interior +/// mutability. +pub unsafe trait AsBytes { + /// Returns `self` as a slice of bytes. + fn as_bytes(&self) -> &[u8] { + // CAST: `Self` implements `AsBytes` thus all bytes of `self` are initialized. + let data = core::ptr::from_ref(self).cast::(); + let len = core::mem::size_of_val(self); + + // SAFETY: `data` is non-null and valid for reads of `len * sizeof::()` bytes. + unsafe { core::slice::from_raw_parts(data, len) } + } + + /// Returns `self` as a mutable slice of bytes. + fn as_bytes_mut(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + // CAST: `Self` implements both `AsBytes` and `FromBytes` thus making `Self` + // bi-directionally transmutable to `[u8; size_of_val(self)]`. + let data = core::ptr::from_mut(self).cast::(); + let len = core::mem::size_of_val(self); + + // SAFETY: `data` is non-null and valid for read and writes of `len * sizeof::()` + // bytes. + unsafe { core::slice::from_raw_parts_mut(data, len) } + } +} + +macro_rules! impl_asbytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? AsBytes for $t {})* + }; +} + +impl_asbytes! { + // SAFETY: Inhabited ZSTs only have one possible bit pattern, and these two have no invariant. + (), + {} core::marker::PhantomData, + + // SAFETY: Instances of the following types have no uninitialized portions. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + bool, + char, + str, + + // SAFETY: If individual values in an array have no uninitialized portions, then the array + // itself does not have any uninitialized portions either. + {} [T], + {} [T; N], +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs new file mode 100644 index 000000000..132dd428c --- /dev/null +++ b/rust/kernel/types.rs @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel types. + +use crate::ffi::c_void; +use core::{ + cell::UnsafeCell, + marker::{PhantomData, PhantomPinned}, + mem::MaybeUninit, + ops::{Deref, DerefMut}, +}; +use pin_init::{PinInit, Wrapper, Zeroable}; + +#[doc(hidden)] +pub mod for_lt; +pub use for_lt::{ + CovariantForLt, + ForLt, // +}; + +/// Used to transfer ownership to and from foreign (non-Rust) languages. +/// +/// Ownership is transferred from Rust to a foreign language by calling [`Self::into_foreign`] and +/// later may be transferred back to Rust by calling [`Self::from_foreign`]. +/// +/// This trait is meant to be used in cases when Rust objects are stored in C objects and +/// eventually "freed" back to Rust. +/// +/// # Safety +/// +/// - Implementations must satisfy the guarantees of [`Self::into_foreign`]. +pub unsafe trait ForeignOwnable: Sized { + /// The alignment of pointers returned by `into_foreign`. + const FOREIGN_ALIGN: usize; + + /// Type used to immutably borrow a value that is currently foreign-owned. + type Borrowed<'a> + where + Self: 'a; + + /// Type used to mutably borrow a value that is currently foreign-owned. + type BorrowedMut<'a> + where + Self: 'a; + + /// Converts a Rust-owned object to a foreign-owned one. + /// + /// The foreign representation is a pointer to void. Aside from the guarantees listed below, + /// there are no other guarantees for this pointer. For example, it might be invalid, dangling + /// or pointing to uninitialized memory. Using it in any way except for [`from_foreign`], + /// [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can result in undefined behavior. + /// + /// # Guarantees + /// + /// - Minimum alignment of returned pointer is [`Self::FOREIGN_ALIGN`]. + /// - The returned pointer is not null. + /// + /// [`from_foreign`]: Self::from_foreign + /// [`try_from_foreign`]: Self::try_from_foreign + /// [`borrow`]: Self::borrow + /// [`borrow_mut`]: Self::borrow_mut + fn into_foreign(self) -> *mut c_void; + + /// Converts a foreign-owned object back to a Rust-owned one. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and it + /// must not be passed to `from_foreign` more than once. + /// + /// [`into_foreign`]: Self::into_foreign + unsafe fn from_foreign(ptr: *mut c_void) -> Self; + + /// Tries to convert a foreign-owned object back to a Rust-owned one. + /// + /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr` + /// is null. + /// + /// # Safety + /// + /// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`]. + /// + /// [`from_foreign`]: Self::from_foreign + unsafe fn try_from_foreign(ptr: *mut c_void) -> Option { + if ptr.is_null() { + None + } else { + // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements + // of `from_foreign` given the safety requirements of this function. + unsafe { Some(Self::from_foreign(ptr)) } + } + } + + /// Borrows a foreign-owned object immutably. + /// + /// This method provides a way to access a foreign-owned value from Rust immutably. It provides + /// you with exactly the same abilities as an `&Self` when the value is Rust-owned. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>; + + /// Borrows a foreign-owned object mutably. + /// + /// This method provides a way to access a foreign-owned value from Rust mutably. It provides + /// you with exactly the same abilities as an `&mut Self` when the value is Rust-owned, except + /// that the address of the object must not be changed. + /// + /// Note that for types like [`Arc`], an `&mut Arc` only gives you immutable access to the + /// inner value, so this method also only provides immutable access in that case. + /// + /// In the case of `Box`, this method gives you the ability to modify the inner `T`, but it + /// does not let you change the box itself. That is, you cannot change which allocation the box + /// points at. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// The lifetime `'a` must not overlap with the lifetime of any other call to [`borrow`] or + /// `borrow_mut` on the same object. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + /// [`borrow`]: Self::borrow + /// [`Arc`]: crate::sync::Arc + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>; +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `()`. +unsafe impl ForeignOwnable for () { + const FOREIGN_ALIGN: usize = core::mem::align_of::<()>(); + type Borrowed<'a> = (); + type BorrowedMut<'a> = (); + + fn into_foreign(self) -> *mut c_void { + core::ptr::NonNull::dangling().as_ptr() + } + + unsafe fn from_foreign(_: *mut c_void) -> Self {} + + unsafe fn borrow<'a>(_: *mut c_void) -> Self::Borrowed<'a> {} + unsafe fn borrow_mut<'a>(_: *mut c_void) -> Self::BorrowedMut<'a> {} +} + +/// Runs a cleanup function/closure when dropped. +/// +/// The [`ScopeGuard::dismiss`] function prevents the cleanup function from running. +/// +/// # Examples +/// +/// In the example below, we have multiple exit paths and we want to log regardless of which one is +/// taken: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example1(arg: bool) { +/// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n")); +/// +/// if arg { +/// return; +/// } +/// +/// pr_info!("Do something...\n"); +/// } +/// +/// # example1(false); +/// # example1(true); +/// ``` +/// +/// In the example below, we want to log the same message on all early exits but a different one on +/// the main exit path: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example2(arg: bool) { +/// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n")); +/// +/// if arg { +/// return; +/// } +/// +/// // (Other early returns...) +/// +/// log.dismiss(); +/// pr_info!("example2 no early return\n"); +/// } +/// +/// # example2(false); +/// # example2(true); +/// ``` +/// +/// In the example below, we need a mutable object (the vector) to be accessible within the log +/// function, so we wrap it in the [`ScopeGuard`]: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example3(arg: bool) -> Result { +/// let mut vec = +/// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); +/// +/// vec.push(10u8, GFP_KERNEL)?; +/// if arg { +/// return Ok(()); +/// } +/// vec.push(20u8, GFP_KERNEL)?; +/// Ok(()) +/// } +/// +/// # assert_eq!(example3(false), Ok(())); +/// # assert_eq!(example3(true), Ok(())); +/// ``` +/// +/// # Invariants +/// +/// The value stored in the struct is nearly always `Some(_)`, except between +/// [`ScopeGuard::dismiss`] and [`ScopeGuard::drop`]: in this case, it will be `None` as the value +/// will have been returned to the caller. Since [`ScopeGuard::dismiss`] consumes the guard, +/// callers won't be able to use it anymore. +pub struct ScopeGuard(Option<(T, F)>); + +impl ScopeGuard { + /// Creates a new guarded object wrapping the given data and with the given cleanup function. + pub fn new_with_data(data: T, cleanup_func: F) -> Self { + // INVARIANT: The struct is being initialised with `Some(_)`. + Self(Some((data, cleanup_func))) + } + + /// Prevents the cleanup function from running and returns the guarded data. + pub fn dismiss(mut self) -> T { + // INVARIANT: This is the exception case in the invariant; it is not visible to callers + // because this function consumes `self`. + self.0.take().unwrap().0 + } +} + +impl ScopeGuard<(), fn(())> { + /// Creates a new guarded object with the given cleanup function. + pub fn new(cleanup: impl FnOnce()) -> ScopeGuard<(), impl FnOnce(())> { + ScopeGuard::new_with_data((), move |()| cleanup()) + } +} + +impl Deref for ScopeGuard { + type Target = T; + + fn deref(&self) -> &T { + // The type invariants guarantee that `unwrap` will succeed. + &self.0.as_ref().unwrap().0 + } +} + +impl DerefMut for ScopeGuard { + fn deref_mut(&mut self) -> &mut T { + // The type invariants guarantee that `unwrap` will succeed. + &mut self.0.as_mut().unwrap().0 + } +} + +impl Drop for ScopeGuard { + fn drop(&mut self) { + // Run the cleanup function if one is still present. + if let Some((data, cleanup)) = self.0.take() { + cleanup(data) + } + } +} + +/// Stores an opaque value. +/// +/// [`Opaque`] is meant to be used with FFI objects that are never interpreted by Rust code. +/// +/// It is used to wrap structs from the C side, like for example `Opaque`. +/// It gets rid of all the usual assumptions that Rust has for a value: +/// +/// * The value is allowed to be uninitialized (for example have invalid bit patterns: `3` for a +/// [`bool`]). +/// * The value is allowed to be mutated, when a `&Opaque` exists on the Rust side. +/// * No uniqueness for mutable references: it is fine to have multiple `&mut Opaque` point to +/// the same value. +/// * The value is not allowed to be shared with other threads (i.e. it is `!Sync`). +/// +/// This has to be used for all values that the C side has access to, because it can't be ensured +/// that the C side is adhering to the usual constraints that Rust needs. +/// +/// Using [`Opaque`] allows to continue to use references on the Rust side even for values shared +/// with C. +/// +/// # Examples +/// +/// ``` +/// use kernel::types::Opaque; +/// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side +/// # // knows. +/// # mod bindings { +/// # pub struct Foo { +/// # pub val: u8, +/// # } +/// # } +/// +/// // `foo.val` is assumed to be handled on the C side, so we use `Opaque` to wrap it. +/// pub struct Foo { +/// foo: Opaque, +/// } +/// +/// impl Foo { +/// pub fn get_val(&self) -> u8 { +/// let ptr = Opaque::get(&self.foo); +/// +/// // SAFETY: `Self` is valid from C side. +/// unsafe { (*ptr).val } +/// } +/// } +/// +/// // Create an instance of `Foo` with the `Opaque` wrapper. +/// let foo = Foo { +/// foo: Opaque::new(bindings::Foo { val: 0xdb }), +/// }; +/// +/// assert_eq!(foo.get_val(), 0xdb); +/// ``` +#[repr(transparent)] +pub struct Opaque { + value: UnsafeCell>, + _pin: PhantomPinned, +} + +// SAFETY: `Opaque` allows the inner value to be any bit pattern, including all zeros. +unsafe impl Zeroable for Opaque {} + +impl Opaque { + /// Creates a new opaque value. + pub const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::new(value)), + _pin: PhantomPinned, + } + } + + /// Creates an uninitialised value. + pub const fn uninit() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + _pin: PhantomPinned, + } + } + + /// Creates a new zeroed opaque value. + pub const fn zeroed() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::zeroed()), + _pin: PhantomPinned, + } + } + + /// Creates a pin-initializer from the given initializer closure. + /// + /// The returned initializer calls the given closure with the pointer to the inner `T` of this + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. + /// + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs + /// to verify at that point that the inner value is valid. + pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit { + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully + // initialize the `T`. + unsafe { + pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { + init_func(Self::cast_into(slot)); + Ok(()) + }) + } + } + + /// Creates a fallible pin-initializer from the given initializer closure. + /// + /// The returned initializer calls the given closure with the pointer to the inner `T` of this + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. + /// + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs + /// to verify at that point that the inner value is valid. + pub fn try_ffi_init( + init_func: impl FnOnce(*mut T) -> Result<(), E>, + ) -> impl PinInit { + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully + // initialize the `T`. + unsafe { + pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::cast_into(slot))) + } + } + + /// Returns a raw pointer to the opaque data. + pub const fn get(&self) -> *mut T { + UnsafeCell::get(&self.value).cast::() + } + + /// Gets the value behind `this`. + /// + /// This function is useful to get access to the value without creating intermediate + /// references. + pub const fn cast_into(this: *const Self) -> *mut T { + UnsafeCell::raw_get(this.cast::>>()).cast::() + } + + /// The opposite operation of [`Opaque::cast_into`]. + pub const fn cast_from(this: *const T) -> *const Self { + this.cast() + } +} + +impl Wrapper for Opaque { + /// Create an opaque pin-initializer from the given pin-initializer. + fn pin_init(init: impl PinInit) -> impl PinInit { + Self::try_ffi_init(|slot: *mut T| { + // SAFETY: + // - `slot` is a valid pointer to uninitialized memory, + // - `slot` is not accessed on error, + // - `slot` is pinned in memory. + unsafe { pin_init::raw_try_init(slot, init) } + }) + } +} + +/// Zero-sized type to mark types not [`Send`]. +/// +/// Add this type as a field to your struct if your type should not be sent to a different task. +/// Since [`Send`] is an auto trait, adding a single field that is `!Send` will ensure that the +/// whole type is `!Send`. +/// +/// If a type is `!Send` it is impossible to give control over an instance of the type to another +/// task. This is useful to include in types that store or reference task-local information. A file +/// descriptor is an example of such task-local information. +/// +/// This type also makes the type `!Sync`, which prevents immutable access to the value from +/// several threads in parallel. +pub type NotThreadSafe = PhantomData<*mut ()>; + +/// Used to construct instances of type [`NotThreadSafe`] similar to how `PhantomData` is +/// constructed. +/// +/// [`NotThreadSafe`]: type@NotThreadSafe +#[allow(non_upper_case_globals)] +pub const NotThreadSafe: NotThreadSafe = PhantomData; diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs new file mode 100644 index 000000000..b8f422c80 --- /dev/null +++ b/rust/kernel/types/for_lt.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and +//! macros. +//! +//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) / +//! [`CovariantForLt!`](macro@CovariantForLt) directly. + +use core::marker::PhantomData; + +/// Representation of types generic over a lifetime. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is +/// provided to obtain a type that implements this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that ` TypeThatUse<'a>) as ForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(TypeThatUse<'_>) // Equivalent to `ForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(&u32) // Equivalent to `ForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. +/// # >(); +/// ``` +pub trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; +} +pub use macros::ForLt; + +/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter. +/// +/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be +/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. +/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements +/// this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that +/// ` TypeThatUse<'a>) as CovariantForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`. +/// # >(); +/// ``` +/// +/// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. +/// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove +/// it. +/// +/// ```ignore,compile_fail +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # expect_lt::< +/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// # >(); +/// ``` +/// +/// There is a limitation if the type refers to generic parameters; if the macro cannot prove the +/// covariance syntactically, the emitted checks will fail the compilation as it needs to refer to +/// the generic parameter but is in a separate item. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// fn expect_lt() {} +/// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] +/// fn generic_fn() { +/// // Syntactically proven by the macro +/// expect_lt::(); +/// // Syntactically proven by the macro +/// expect_lt::)>(); +/// // Cannot be syntactically proven, need to check covariance of `KBox` +/// // expect_lt::)>(); +/// } +/// ``` +/// +/// # Safety +/// +/// `Self::Of<'a>` must be covariant over the lifetime `'a`. +pub unsafe trait CovariantForLt: ForLt { + /// Cast a reference to a shorter lifetime. + #[inline(always)] + fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { + // SAFETY: This is sound as this trait guarantees covariance. + unsafe { core::mem::transmute(long) } + } +} +pub use macros::CovariantForLt; + +/// This is intended to be an "unsafe-to-refer-to" type. +/// +/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt) +/// macros. +/// +/// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. +/// +/// `WF` is a type that the macro can use to assert some specific type is well-formed. +/// +/// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove +/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`; +/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof. +#[doc(hidden)] +pub struct UnsafeForLtImpl(PhantomData<(WF, T)>); + +// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB. +#[doc(hidden)] +pub trait WithLt<'a> { + type Of: 'a; +} + +impl WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl { + type Of<'a> = >::Of; +} + +// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic +// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl. +unsafe impl WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl {} diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs new file mode 100644 index 000000000..5f6c4d7a1 --- /dev/null +++ b/rust/kernel/uaccess.rs @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Slices to user space memory regions. +//! +//! C header: [`include/linux/uaccess.h`](srctree/include/linux/uaccess.h) + +use crate::{ + alloc::{Allocator, Flags}, + bindings, + dma::Coherent, + error::Result, + ffi::{c_char, c_void}, + fs::file, + prelude::*, + ptr::KnownSize, + transmute::{AsBytes, FromBytes}, +}; +use core::mem::{size_of, MaybeUninit}; + +/// A pointer into userspace. +/// +/// This is the Rust equivalent to C pointers tagged with `__user`. +#[repr(transparent)] +#[derive(Copy, Clone, Zeroable)] +pub struct UserPtr(*mut c_void); + +impl UserPtr { + /// Create a `UserPtr` from an integer representing the userspace address. + #[inline] + pub fn from_addr(addr: usize) -> Self { + Self(addr as *mut c_void) + } + + /// Create a `UserPtr` from a pointer representing the userspace address. + #[inline] + pub fn from_ptr(addr: *mut c_void) -> Self { + Self(addr) + } + + /// Cast this userspace pointer to a raw const void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_const_ptr(self) -> *const c_void { + self.0 + } + + /// Cast this userspace pointer to a raw mutable void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_mut_ptr(self) -> *mut c_void { + self.0 + } + + /// Increment this user pointer by `add` bytes. + /// + /// This addition is wrapping, so wrapping around the address space does not result in a panic + /// even if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled. + #[inline] + pub fn wrapping_byte_add(self, add: usize) -> UserPtr { + UserPtr(self.0.wrapping_byte_add(add)) + } +} + +/// A pointer to an area in userspace memory, which can be either read-only or read-write. +/// +/// All methods on this struct are safe: attempting to read or write on bad addresses (either out of +/// the bound of the slice or unmapped addresses) will return [`EFAULT`]. Concurrent access, +/// *including data races to/from userspace memory*, is permitted, because fundamentally another +/// userspace thread/process could always be modifying memory at the same time (in the same way that +/// userspace Rust's [`std::io`] permits data races with the contents of files on disk). In the +/// presence of a race, the exact byte values read/written are unspecified but the operation is +/// well-defined. Kernelspace code should validate its copy of data after completing a read, and not +/// expect that multiple reads of the same address will return the same value. +/// +/// These APIs are designed to make it difficult to accidentally write TOCTOU (time-of-check to +/// time-of-use) bugs. Every time a memory location is read, the reader's position is advanced by +/// the read length and the next read will start from there. This helps prevent accidentally reading +/// the same location twice and causing a TOCTOU bug. +/// +/// Creating a [`UserSliceReader`] and/or [`UserSliceWriter`] consumes the `UserSlice`, helping +/// ensure that there aren't multiple readers or writers to the same location. +/// +/// If double-fetching a memory location is necessary for some reason, then that is done by creating +/// multiple readers to the same memory location, e.g. using [`clone_reader`]. +/// +/// # Examples +/// +/// Takes a region of userspace memory from the current process, and modify it by adding one to +/// every byte in the region. +/// +/// ```no_run +/// use kernel::ffi::c_void; +/// use kernel::uaccess::{UserPtr, UserSlice}; +/// +/// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result { +/// let (read, mut write) = UserSlice::new(uptr, len).reader_writer(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// for b in &mut buf { +/// *b = b.wrapping_add(1); +/// } +/// +/// write.write_slice(&buf)?; +/// Ok(()) +/// } +/// ``` +/// +/// Example illustrating a TOCTOU (time-of-check to time-of-use) bug. +/// +/// ```no_run +/// use kernel::ffi::c_void; +/// use kernel::uaccess::{UserPtr, UserSlice}; +/// +/// /// Returns whether the data in this region is valid. +/// fn is_valid(uptr: UserPtr, len: usize) -> Result { +/// let read = UserSlice::new(uptr, len).reader(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// todo!() +/// } +/// +/// /// Returns the bytes behind this user pointer if they are valid. +/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { +/// if !is_valid(uptr, len)? { +/// return Err(EINVAL); +/// } +/// +/// let read = UserSlice::new(uptr, len).reader(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// // THIS IS A BUG! The bytes could have changed since we checked them. +/// // +/// // To avoid this kind of bug, don't call `UserSlice::new` multiple +/// // times with the same address. +/// Ok(buf) +/// } +/// ``` +/// +/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html +/// [`clone_reader`]: UserSliceReader::clone_reader +pub struct UserSlice { + ptr: UserPtr, + length: usize, +} + +impl UserSlice { + /// Constructs a user slice from a raw pointer and a length in bytes. + /// + /// Constructing a [`UserSlice`] performs no checks on the provided address and length, it can + /// safely be constructed inside a kernel thread with no current userspace process. Reads and + /// writes wrap the kernel APIs `copy_from_user` and `copy_to_user`, which check the memory map + /// of the current process and enforce that the address range is within the user range (no + /// additional calls to `access_ok` are needed). Validity of the pointer is checked when you + /// attempt to read or write, not in the call to `UserSlice::new`. + /// + /// Callers must be careful to avoid time-of-check-time-of-use (TOCTOU) issues. The simplest way + /// is to create a single instance of [`UserSlice`] per user memory block as it reads each byte + /// at most once. + pub fn new(ptr: UserPtr, length: usize) -> Self { + UserSlice { ptr, length } + } + + /// Reads the entirety of the user slice, appending it to the end of the provided buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address. + pub fn read_all(self, buf: &mut Vec, flags: Flags) -> Result { + self.reader().read_all(buf, flags) + } + + /// Constructs a [`UserSliceReader`]. + pub fn reader(self) -> UserSliceReader { + UserSliceReader { + ptr: self.ptr, + length: self.length, + } + } + + /// Constructs a [`UserSliceWriter`]. + pub fn writer(self) -> UserSliceWriter { + UserSliceWriter { + ptr: self.ptr, + length: self.length, + } + } + + /// Constructs both a [`UserSliceReader`] and a [`UserSliceWriter`]. + /// + /// Usually when this is used, you will first read the data, and then overwrite it afterwards. + pub fn reader_writer(self) -> (UserSliceReader, UserSliceWriter) { + ( + UserSliceReader { + ptr: self.ptr, + length: self.length, + }, + UserSliceWriter { + ptr: self.ptr, + length: self.length, + }, + ) + } +} + +/// A reader for [`UserSlice`]. +/// +/// Used to incrementally read from the user slice. +pub struct UserSliceReader { + ptr: UserPtr, + length: usize, +} + +impl UserSliceReader { + /// Skip the provided number of bytes. + /// + /// Returns an error if skipping more than the length of the buffer. + pub fn skip(&mut self, num_skip: usize) -> Result { + // Update `self.length` first since that's the fallible part of this operation. + self.length = self.length.checked_sub(num_skip).ok_or(EFAULT)?; + self.ptr = self.ptr.wrapping_byte_add(num_skip); + Ok(()) + } + + /// Create a reader that can access the same range of data. + /// + /// Reading from the clone does not advance the current reader. + /// + /// The caller should take care to not introduce TOCTOU issues, as described in the + /// documentation for [`UserSlice`]. + pub fn clone_reader(&self) -> UserSliceReader { + UserSliceReader { + ptr: self.ptr, + length: self.length, + } + } + + /// Returns the number of bytes left to be read from this reader. + /// + /// Note that even reading less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.length + } + + /// Returns `true` if no data is available in the io buffer. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Reads raw data from the user slice into a kernel buffer. + /// + /// For a version that uses `&mut [u8]`, please see [`UserSliceReader::read_slice`]. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. + /// + /// # Guarantees + /// + /// After a successful call to this method, all bytes in `out` are initialized. + pub fn read_raw(&mut self, out: &mut [MaybeUninit]) -> Result { + let len = out.len(); + let out_ptr = out.as_mut_ptr().cast::(); + if len > self.length { + return Err(EFAULT); + } + // SAFETY: `out_ptr` points into a mutable slice of length `len`, so we may write + // that many bytes to it. + let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr.as_const_ptr(), len) }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } + + /// Reads raw data from the user slice into a kernel buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. + pub fn read_slice(&mut self, out: &mut [u8]) -> Result { + // SAFETY: The types are compatible and `read_raw` doesn't write uninitialized bytes to + // `out`. + let out = unsafe { &mut *(core::ptr::from_mut(out) as *mut [MaybeUninit]) }; + self.read_raw(out) + } + + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice`] but considers the given `offset` into `out` and + /// truncates the read to the boundaries of `self` and `out`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_partial(&mut self, out: &mut [u8], offset: usize) -> Result { + let end = offset.saturating_add(self.len()).min(out.len()); + + let Some(dst) = out.get_mut(offset..end) else { + return Ok(0); + }; + + self.read_slice(dst)?; + Ok(dst.len()) + } + + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes read. + /// + /// This is equivalent to C's `simple_write_to_buffer()`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_file(&mut self, out: &mut [u8], offset: &mut file::Offset) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let read = self.read_slice_partial(out, offset_index)?; + + // OVERFLOW: `offset + read <= data.len() <= isize::MAX <= Offset::MAX` + *offset += read as i64; + + Ok(read) + } + + /// Reads a value of the specified type. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. + pub fn read(&mut self) -> Result { + let len = size_of::(); + if len > self.length { + return Err(EFAULT); + } + let mut out: MaybeUninit = MaybeUninit::uninit(); + // SAFETY: The local variable `out` is valid for writing `size_of::()` bytes. + // + // By using the _copy_from_user variant, we skip the check_object_size check that verifies + // the kernel pointer. This mirrors the logic on the C side that skips the check when the + // length is a compile-time constant. + let res = unsafe { + bindings::_copy_from_user( + out.as_mut_ptr().cast::(), + self.ptr.as_const_ptr(), + len, + ) + }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + // SAFETY: The read above has initialized all bytes in `out`, and since `T` implements + // `FromBytes`, any bit-pattern is a valid value for this type. + Ok(unsafe { out.assume_init() }) + } + + /// Reads the entirety of the user slice, appending it to the end of the provided buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address. + pub fn read_all(mut self, buf: &mut Vec, flags: Flags) -> Result { + let len = self.length; + buf.reserve(len, flags)?; + + // The call to `reserve` was successful, so the spare capacity is at least `len` bytes long. + self.read_raw(&mut buf.spare_capacity_mut()[..len])?; + + // SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the + // vector have been initialized. + unsafe { buf.inc_len(len) }; + Ok(()) + } + + /// Read a NUL-terminated string from userspace and return it. + /// + /// The string is read into `buf` and a NUL-terminator is added if the end of `buf` is reached. + /// Since there must be space to add a NUL-terminator, the buffer must not be empty. The + /// returned `&CStr` points into `buf`. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address (some data may have been + /// copied). + #[doc(alias = "strncpy_from_user")] + pub fn strcpy_into_buf<'buf>(self, buf: &'buf mut [u8]) -> Result<&'buf CStr> { + if buf.is_empty() { + return Err(EINVAL); + } + + // SAFETY: The types are compatible and `strncpy_from_user` doesn't write uninitialized + // bytes to `buf`. + let mut dst = unsafe { &mut *(core::ptr::from_mut(buf) as *mut [MaybeUninit]) }; + + // We never read more than `self.length` bytes. + if dst.len() > self.length { + dst = &mut dst[..self.length]; + } + + let mut len = raw_strncpy_from_user(dst, self.ptr)?; + if len < dst.len() { + // Add one to include the NUL-terminator. + len += 1; + } else if len < buf.len() { + // This implies that `len == dst.len() < buf.len()`. + // + // This means that we could not fill the entire buffer, but we had to stop reading + // because we hit the `self.length` limit of this `UserSliceReader`. Since we did not + // fill the buffer, we treat this case as if we tried to read past the `self.length` + // limit and received a page fault, which is consistent with other `UserSliceReader` + // methods that also return page faults when you exceed `self.length`. + return Err(EFAULT); + } else { + // This implies that `len == buf.len()`. + // + // This means that we filled the buffer exactly. In this case, we add a NUL-terminator + // and return it. Unlike the `len < dst.len()` branch, don't modify `len` because it + // already represents the length including the NUL-terminator. + // + // SAFETY: Due to the check at the beginning, the buffer is not empty. + unsafe { *buf.last_mut().unwrap_unchecked() = 0 }; + } + + // This method consumes `self`, so it can only be called once, thus we do not need to + // update `self.length`. This sidesteps concerns such as whether `self.length` should be + // incremented by `len` or `len-1` in the `len == buf.len()` case. + + // SAFETY: There are two cases: + // * If we hit the `len < dst.len()` case, then `raw_strncpy_from_user` guarantees that + // this slice contains exactly one NUL byte at the end of the string. + // * Otherwise, `raw_strncpy_from_user` guarantees that the string contained no NUL bytes, + // and we have since added a NUL byte at the end. + Ok(unsafe { CStr::from_bytes_with_nul_unchecked(&buf[..len]) }) + } +} + +/// A writer for [`UserSlice`]. +/// +/// Used to incrementally write into the user slice. +pub struct UserSliceWriter { + ptr: UserPtr, + length: usize, +} + +impl UserSliceWriter { + /// Returns the amount of space remaining in this buffer. + /// + /// Note that even writing less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.length + } + + /// Returns `true` if no more data can be written to this buffer. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Low-level write from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `from` is valid for reads of `len` bytes. + unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { + if len > self.length { + return Err(EFAULT); + } + + // SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's + // safety contract). + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } + + /// Writes raw data to this user pointer from a kernel buffer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for + // reading `data.len()` bytes. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to this user pointer from a DMA coherent allocation. + /// + /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. + /// + /// # Errors + /// + /// - [`EOVERFLOW`]: `offset + count` overflows. + /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the + /// size of the user-space buffer. + /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this + /// [`UserSliceWriter`]. + /// + /// This call may modify the associated userspace slice even if it returns an error. + /// + /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, + /// the copied data may be inconsistent, but this does not cause undefined behavior. + /// + /// # Example + /// + /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: + /// + /// ```no_run + /// use kernel::uaccess::UserSliceWriter; + /// use kernel::dma::Coherent; + /// + /// fn copy_dma_to_user( + /// mut writer: UserSliceWriter, + /// alloc: &Coherent<[u8]>, + /// ) -> Result { + /// writer.write_dma(alloc, 0, 256) + /// } + /// ``` + pub fn write_dma( + &mut self, + alloc: &Coherent, + offset: usize, + count: usize, + ) -> Result { + let len = alloc.size(); + if offset.checked_add(count).ok_or(EOVERFLOW)? > len { + return Err(ERANGE); + } + + if count > self.len() { + return Err(ERANGE); + } + + // SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as + // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. + let src_ptr = unsafe { alloc.as_ptr().cast::().add(offset) }; + + // Note: Use `write_raw` instead of `write_slice` because the allocation is coherent + // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over + // such volatile memory. + // + // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). + unsafe { self.write_raw(src_ptr, count) } + } + + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and + /// truncates the write to the boundaries of `self` and `data`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result { + let end = offset.saturating_add(self.len()).min(data.len()); + + let Some(src) = data.get(offset..end) else { + return Ok(0); + }; + + self.write_slice(src)?; + Ok(src.len()) + } + + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes written. + /// + /// This is equivalent to C's `simple_read_from_buffer()`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_file(&mut self, data: &[u8], offset: &mut file::Offset) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let written = self.write_slice_partial(data, offset_index)?; + + // OVERFLOW: `offset + written <= data.len() <= isize::MAX <= Offset::MAX` + *offset += written as i64; + + Ok(written) + } + + /// Writes the provided Rust value to this userspace pointer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write(&mut self, value: &T) -> Result { + let len = size_of::(); + if len > self.length { + return Err(EFAULT); + } + // SAFETY: The reference points to a value of type `T`, so it is valid for reading + // `size_of::()` bytes. + // + // By using the _copy_to_user variant, we skip the check_object_size check that verifies the + // kernel pointer. This mirrors the logic on the C side that skips the check when the length + // is a compile-time constant. + let res = unsafe { + bindings::_copy_to_user( + self.ptr.as_mut_ptr(), + core::ptr::from_ref(value).cast::(), + len, + ) + }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } +} + +/// Reads a nul-terminated string into `dst` and returns the length. +/// +/// This reads from userspace until a NUL byte is encountered, or until `dst.len()` bytes have been +/// read. Fails with [`EFAULT`] if a read happens on a bad address (some data may have been +/// copied). When the end of the buffer is encountered, no NUL byte is added, so the string is +/// *not* guaranteed to be NUL-terminated when `Ok(dst.len())` is returned. +/// +/// # Guarantees +/// +/// When this function returns `Ok(len)`, it is guaranteed that the first `len` bytes of `dst` are +/// initialized and non-zero. Furthermore, if `len < dst.len()`, then `dst[len]` is a NUL byte. +#[inline] +fn raw_strncpy_from_user(dst: &mut [MaybeUninit], src: UserPtr) -> Result { + // CAST: Slice lengths are guaranteed to be `<= isize::MAX`. + let len = dst.len() as isize; + + // SAFETY: `dst` is valid for writing `dst.len()` bytes. + let res = unsafe { + bindings::strncpy_from_user( + dst.as_mut_ptr().cast::(), + src.as_const_ptr().cast::(), + len, + ) + }; + + if res < 0 { + return Err(Error::from_errno(res as i32)); + } + + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res <= len); + + // GUARANTEES: `strncpy_from_user` was successful, so `dst` has contents in accordance with the + // guarantees of this function. + Ok(res as usize) +} diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs new file mode 100644 index 000000000..993760ff3 --- /dev/null +++ b/rust/kernel/usb.rs @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (C) 2025 Collabora Ltd. + +//! Abstractions for the USB bus. +//! +//! C header: [`include/linux/usb.h`](srctree/include/linux/usb.h) + +use crate::{ + bindings, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + sync::aref::AlwaysRefCounted, + types::Opaque, + ThisModule, // +}; +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::NonNull, // +}; + +/// An adapter for the registration of USB drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::usb_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct usb_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::usb_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + udrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct usb_driver` on initialization. + unsafe { + (*udrv.get()).name = name.as_char_ptr(); + (*udrv.get()).probe = Some(Self::probe_callback); + (*udrv.get()).disconnect = Some(Self::disconnect_callback); + (*udrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::usb_register_driver(udrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(udrv: &Opaque) { + // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::usb_deregister(udrv.get()) }; + } +} + +impl Adapter { + extern "C" fn probe_callback( + intf: *mut bindings::usb_interface, + id: *const bindings::usb_device_id, + ) -> kernel::ffi::c_int { + // SAFETY: The USB core only ever calls the probe callback with a valid pointer to a + // `struct usb_interface` and `struct usb_device_id`. + // + // INVARIANT: `intf` is valid for the duration of `probe_callback()`. + let intf = unsafe { &*intf.cast::>>() }; + + from_result(|| { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct usb_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. It + // can also come from dynamic IDs, which will ensure that `driver_data` exists in + // `T::ID_TABLE` or is 0. + let info = unsafe { id.info_unchecked_opt::() }; + let data = T::probe(intf, id, info); + + let dev: &device::Device> = intf.as_ref(); + dev.set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) { + // SAFETY: The USB core only ever calls the disconnect callback with a valid pointer to a + // `struct usb_interface`. + // + // INVARIANT: `intf` is valid for the duration of `disconnect_callback()`. + let intf = unsafe { &*intf.cast::>>() }; + + let dev: &device::Device> = intf.as_ref(); + + // SAFETY: `disconnect_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { dev.drvdata_borrow::>() }; + + T::disconnect(intf, data); + } +} + +/// Abstraction for the USB device ID structure, i.e. [`struct usb_device_id`]. +/// +/// [`struct usb_device_id`]: https://docs.kernel.org/driver-api/basics.html#c.usb_device_id +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::usb_device_id); + +impl DeviceId { + /// Equivalent to C's `USB_DEVICE` macro. + pub const fn from_id(vendor: u16, product: u16) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16, + idVendor: vendor, + idProduct: product, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_VER` macro. + pub const fn from_device_ver(vendor: u16, product: u16, bcd_lo: u16, bcd_hi: u16) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION as u16, + idVendor: vendor, + idProduct: product, + bcdDevice_lo: bcd_lo, + bcdDevice_hi: bcd_hi, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INFO` macro. + pub const fn from_device_info(class: u8, subclass: u8, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEV_INFO as u16, + bDeviceClass: class, + bDeviceSubClass: subclass, + bDeviceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_INTERFACE_INFO` macro. + pub const fn from_interface_info(class: u8, subclass: u8, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_INT_INFO as u16, + bInterfaceClass: class, + bInterfaceSubClass: subclass, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_CLASS` macro. + pub const fn from_device_interface_class(vendor: u16, product: u16, class: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_CLASS) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceClass: class, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_PROTOCOL` macro. + pub const fn from_device_interface_protocol(vendor: u16, product: u16, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_PROTOCOL) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_NUMBER` macro. + pub const fn from_device_interface_number(vendor: u16, product: u16, number: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_NUMBER) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceNumber: number, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_AND_INTERFACE_INFO` macro. + pub const fn from_device_and_interface_info( + vendor: u16, + product: u16, + class: u8, + subclass: u8, + protocol: u8, + ) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_INT_INFO + | bindings::USB_DEVICE_ID_MATCH_DEVICE) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceClass: class, + bInterfaceSubClass: subclass, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `usb_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::usb_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info); +} + +/// [`IdTable`](kernel::device_id::IdTable) type for USB. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a USB `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! usb_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("usb", $crate::usb::DeviceId, $($tt)*); + }; +} + +/// The USB driver trait. +/// +/// # Examples +/// +///``` +/// # use kernel::{bindings, device::Core, usb}; +/// use kernel::prelude::*; +/// +/// struct MyDriver; +/// +/// kernel::usb_device_table!( +/// USB_TABLE, +/// ::IdInfo, +/// [ +/// (usb::DeviceId::from_id(0x1234, 0x5678), ()), +/// (usb::DeviceId::from_id(0xabcd, 0xef01), ()), +/// ] +/// ); +/// +/// impl usb::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const ID_TABLE: usb::IdTable = &USB_TABLE; +/// +/// fn probe<'bound>( +/// _interface: &'bound usb::Interface>, +/// _id: &usb::DeviceId, +/// _info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// +/// fn disconnect<'bound>( +/// _interface: &'bound usb::Interface>, +/// _data: Pin<&Self::Data<'bound>>, +/// ) { +/// } +/// } +///``` +pub trait Driver { + /// The type holding information about each one of the device ids supported by the driver. + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// USB driver probe. + /// + /// Called when a new USB interface is bound to this driver. + /// Implementers should attempt to initialize the interface here. + fn probe<'bound>( + interface: &'bound Interface>, + id: &DeviceId, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// USB driver disconnect. + /// + /// Called when the USB interface is about to be unbound from this driver. + fn disconnect<'bound>( + interface: &'bound Interface>, + data: Pin<&Self::Data<'bound>>, + ); +} + +/// A USB interface. +/// +/// This structure represents the Rust abstraction for a C [`struct usb_interface`]. +/// The implementation abstracts the usage of a C [`struct usb_interface`] passed +/// in from the C side. +/// +/// # Invariants +/// +/// An [`Interface`] instance represents a valid [`struct usb_interface`] created +/// by the C portion of the kernel. +/// +/// [`struct usb_interface`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_interface +#[repr(transparent)] +pub struct Interface( + Opaque, + PhantomData, +); + +impl Interface { + fn as_raw(&self) -> *mut bindings::usb_interface { + self.0.get() + } +} + +// SAFETY: `usb::Interface` is a transparent wrapper of `struct usb_interface`. +// The offset is guaranteed to point to a valid device field inside `usb::Interface`. +unsafe impl device::AsBusDevice for Interface { + const OFFSET: usize = offset_of!(bindings::usb_interface, dev); +} + +// SAFETY: `Interface` is a transparent wrapper of a type that doesn't depend on +// `Interface`'s generic argument. +kernel::impl_device_context_deref!(unsafe { Interface }); +kernel::impl_device_context_into_aref!(Interface); + +impl AsRef> for Interface { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct usb_interface`. + let dev = unsafe { &raw mut ((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl AsRef for Interface { + fn as_ref(&self) -> &Device { + // SAFETY: `self.as_raw()` is valid by the type invariants. + let usb_dev = unsafe { bindings::interface_to_usbdev(self.as_raw()) }; + + // SAFETY: For a valid `struct usb_interface` pointer, the above call to + // `interface_to_usbdev()` guarantees to return a valid pointer to a `struct usb_device`. + unsafe { &*(usb_dev.cast()) } + } +} + +// SAFETY: Instances of `Interface` are always reference-counted. +unsafe impl AlwaysRefCounted for Interface { + #[inline] + fn inc_ref(&self) { + // SAFETY: The invariants of `Interface` guarantee that `self.as_raw()` + // returns a valid `struct usb_interface` pointer, for which we will + // acquire a new refcount. + unsafe { bindings::usb_get_intf(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::usb_put_intf(obj.cast().as_ptr()) } + } +} + +// SAFETY: A `Interface` is always reference-counted and can be released from any thread. +unsafe impl Send for Interface {} + +// SAFETY: It is safe to send a &Interface to another thread because we do not +// allow any mutation through a shared reference. +unsafe impl Sync for Interface {} + +/// A USB device. +/// +/// This structure represents the Rust abstraction for a C [`struct usb_device`]. +/// The implementation abstracts the usage of a C [`struct usb_device`] passed in +/// from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid [`struct usb_device`] created by the C portion of the +/// kernel. +/// +/// [`struct usb_device`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_device +#[repr(transparent)] +struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::usb_device { + self.0.get() + } +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: The invariants of `Device` guarantee that `self.as_raw()` + // returns a valid `struct usb_device` pointer, for which we will + // acquire a new refcount. + unsafe { bindings::usb_get_dev(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::usb_put_dev(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct usb_device`. + let dev = unsafe { &raw mut ((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: It is safe to send a &Device to another thread because we do not +// allow any mutation through a shared reference. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct usb_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Declares a kernel module that exposes a single USB driver. +/// +/// # Examples +/// +/// ```ignore +/// module_usb_driver! { +/// type: MyDriver, +/// name: "Module name", +/// author: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_usb_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::usb::Adapter, { $($f)* }); + } +} diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs new file mode 100644 index 000000000..7e253b6f2 --- /dev/null +++ b/rust/kernel/workqueue.rs @@ -0,0 +1,1114 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Work queues. +//! +//! This file has two components: The raw work item API, and the safe work item API. +//! +//! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single +//! type to define multiple `work_struct` fields. This is done by choosing an id for each field, +//! and using that id to specify which field you wish to use. (The actual value doesn't matter, as +//! long as you use different values for different fields of the same struct.) Since these IDs are +//! generic, they are used only at compile-time, so they shouldn't exist in the final binary. +//! +//! # The raw API +//! +//! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an +//! arbitrary function that knows how to enqueue the work item. It should usually not be used +//! directly, but if you want to, you can use it without using the pieces from the safe API. +//! +//! # The safe API +//! +//! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also +//! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user. +//! +//! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type. +//! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue. +//! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something +//! that implements [`WorkItem`]. +//! +//! ## Examples +//! +//! This example defines a struct that holds an integer and can be scheduled on the workqueue. When +//! the struct is executed, it will print the integer. Since there is only one `work_struct` field, +//! we do not need to specify ids for the fields. +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: Work, +//! } +//! +//! impl_has_work! { +//! impl HasWork for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value, +//! work <- new_work!("MyStruct::work"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed. +//! fn print_later(val: Arc) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! # print_later(MyStruct::new(42).unwrap()); +//! ``` +//! +//! The following example shows how multiple `work_struct` fields can be used: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value_1: i32, +//! value_2: i32, +//! #[pin] +//! work_1: Work, +//! #[pin] +//! work_2: Work, +//! } +//! +//! impl_has_work! { +//! impl HasWork for MyStruct { self.work_1 } +//! impl HasWork for MyStruct { self.work_2 } +//! } +//! +//! impl MyStruct { +//! fn new(value_1: i32, value_2: i32) -> Result> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value_1, +//! value_2, +//! work_1 <- new_work!("MyStruct::work_1"), +//! work_2 <- new_work!("MyStruct::work_2"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem<1> for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value_1); +//! } +//! } +//! +//! impl WorkItem<2> for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The second value is: {}\n", this.value_2); +//! } +//! } +//! +//! fn print_1_later(val: Arc) { +//! let _ = workqueue::system().enqueue::, 1>(val); +//! } +//! +//! fn print_2_later(val: Arc) { +//! let _ = workqueue::system().enqueue::, 2>(val); +//! } +//! # print_1_later(MyStruct::new(24, 25).unwrap()); +//! # print_2_later(MyStruct::new(41, 42).unwrap()); +//! ``` +//! +//! This example shows how you can schedule delayed work items: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: DelayedWork, +//! } +//! +//! impl_has_delayed_work! { +//! impl HasDelayedWork for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result> { +//! Arc::pin_init( +//! pin_init!(MyStruct { +//! value, +//! work <- new_delayed_work!("MyStruct::work"), +//! }), +//! GFP_KERNEL, +//! ) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed 12 jiffies later. +//! fn print_later(val: Arc) { +//! let _ = workqueue::system().enqueue_delayed(val, 12); +//! } +//! +//! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This +//! /// is equivalent to calling `enqueue_delayed` with a delay of zero. +//! fn print_now(val: Arc) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! # print_later(MyStruct::new(42).unwrap()); +//! # print_now(MyStruct::new(42).unwrap()); +//! ``` +//! +//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) + +use crate::{ + alloc::{AllocError, Flags}, + container_of, + prelude::*, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + Arc, + LockClassKey, // + }, + time::Jiffies, + types::Opaque, +}; +use core::{marker::PhantomData, ptr::NonNull}; + +/// Creates a [`Work`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_work { + ($($name:literal)?) => { + $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_work; + +/// Creates a [`DelayedWork`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_delayed_work { + () => { + $crate::workqueue::DelayedWork::new( + $crate::optional_name!(), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!( + ::core::file!(), + ":", + ::core::line!(), + "_timer" + )), + $crate::static_lock_class!(), + ) + }; + ($name:literal) => { + $crate::workqueue::DelayedWork::new( + $crate::c_str!($name), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!($name, "_timer")), + $crate::static_lock_class!(), + ) + }; +} +pub use new_delayed_work; + +/// A kernel work queue. +/// +/// Wraps the kernel's C `struct workqueue_struct`. +/// +/// It allows work items to be queued to run on thread pools managed by the kernel. Several are +/// always available, for example, `system`, `system_highpri`, `system_long`, etc. +#[repr(transparent)] +pub struct Queue(Opaque); + +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Send for Queue {} +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Sync for Queue {} + +impl Queue { + /// Use the provided `struct workqueue_struct` with Rust. + /// + /// # Safety + /// + /// The caller must ensure that the provided raw pointer is not dangling, that it points at a + /// valid workqueue, and that it remains valid until the end of `'a`. + pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { + // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The + // caller promises that the pointer is not dangling. + unsafe { &*ptr.cast::() } + } + + /// Enqueues a work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue(&self, w: W) -> W::EnqueueOutput + where + W: RawWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which + // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this + // closure. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` + // will have returned true. In this case, `__enqueue` promises that the raw pointer will + // stay valid until we call the function pointer in the `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, + queue_ptr, + work_ptr, + ) + }) + } + } + + /// Enqueues a delayed work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue_delayed(&self, w: W, delay: Jiffies) -> W::EnqueueOutput + where + W: RawDelayedWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_delayed_work_on` will dereference the provided raw pointer, + // which is ok because `__enqueue` guarantees that the pointer is valid for the duration of + // this closure, and the safety requirements of `RawDelayedWorkItem` expands this + // requirement to apply to the entire `delayed_work`. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and + // `bindings::queue_delayed_work_on` will have returned true. In this case, `__enqueue` + // promises that the raw pointer will stay valid until we call the function pointer in the + // `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_delayed_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, + queue_ptr, + container_of!(work_ptr, bindings::delayed_work, work), + delay, + ) + }) + } + } + + /// Tries to spawn the given function or closure as a work item. + /// + /// This method can fail because it allocates memory to store the work item. + pub fn try_spawn( + &self, + flags: Flags, + func: T, + ) -> Result<(), AllocError> { + let init = pin_init!(ClosureWork { + work <- new_work!("Queue::try_spawn"), + func: Some(func), + }); + + self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?); + Ok(()) + } +} + +/// A helper type used in [`try_spawn`]. +/// +/// [`try_spawn`]: Queue::try_spawn +#[pin_data] +struct ClosureWork { + #[pin] + work: Work>, + func: Option, +} + +impl WorkItem for ClosureWork { + type Pointer = Pin>; + + fn run(mut this: Pin>) { + if let Some(func) = this.as_mut().project().func.take() { + (func)() + } + } +} + +/// A raw work item. +/// +/// This is the low-level trait that is designed for being as general as possible. +/// +/// The `ID` parameter to this trait exists so that a single type can provide multiple +/// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then +/// you will implement this trait once for each field, using a different id for each field. The +/// actual value of the id is not important as long as you use different ids for different fields +/// of the same struct. (Fields of different structs need not use different ids.) +/// +/// Note that the id is used only to select the right method to call during compilation. It won't be +/// part of the final executable. +/// +/// # Safety +/// +/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`] +/// remain valid for the duration specified in the guarantees section of the documentation for +/// [`__enqueue`]. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +pub unsafe trait RawWorkItem { + /// The return type of [`Queue::enqueue`]. + type EnqueueOutput; + + /// Enqueues this work item on a queue using the provided `queue_work_on` method. + /// + /// # Guarantees + /// + /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a + /// valid `work_struct` for the duration of the call to the closure. If the closure returns + /// true, then it is further guaranteed that the pointer remains valid until someone calls the + /// function pointer stored in the `work_struct`. + /// + /// # Safety + /// + /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. + /// + /// If the work item type is annotated with any lifetimes, then you must not call the function + /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) + /// + /// If the work item type is not [`Send`], then the function pointer must be called on the same + /// thread as the call to `__enqueue`. + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool; +} + +/// A raw delayed work item. +/// +/// # Safety +/// +/// If the `__enqueue` method in the `RawWorkItem` implementation calls the closure, then the +/// provided pointer must point at the `work` field of a valid `delayed_work`, and the guarantees +/// that `__enqueue` provides about accessing the `work_struct` must also apply to the rest of the +/// `delayed_work` struct. +pub unsafe trait RawDelayedWorkItem: RawWorkItem {} + +/// Defines the method that should be called directly when a work item is executed. +/// +/// This trait is implemented by `Pin>`, [`Arc`] and [`ARef`], and +/// is mainly intended to be implemented for smart pointer types. For your own +/// structs, you would implement [`WorkItem`] instead. The [`run`] method on +/// this trait will usually just perform the appropriate `container_of` +/// translation and then call into the [`run`][WorkItem::run] method from the +/// [`WorkItem`] trait. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +/// +/// # Safety +/// +/// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] +/// method of this trait as the function pointer. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +/// [`run`]: WorkItemPointer::run +pub unsafe trait WorkItemPointer: RawWorkItem { + /// Run this work item. + /// + /// # Safety + /// + /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`] + /// where the `queue_work_on` closure returned true, and the pointer must still be valid. + /// + /// [`__enqueue`]: RawWorkItem::__enqueue + unsafe extern "C" fn run(ptr: *mut bindings::work_struct); +} + +/// Defines the method that should be called when this work item is executed. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +pub trait WorkItem { + /// The pointer type that this struct is wrapped in. This will typically be `Arc` or + /// `Pin>`. + type Pointer: WorkItemPointer; + + /// The method that should be called when this work item is executed. + fn run(this: Self::Pointer); +} + +/// Links for a work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. +/// +/// Wraps the kernel's C `struct work_struct`. +/// +/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct Work { + #[pin] + work: Opaque, + _inner: PhantomData, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Send for Work {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Sync for Work {} + +impl Work { + /// Creates a new instance of [`Work`]. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit + where + T: WorkItem, + { + pin_init!(Self { + work <- Opaque::ffi_init(|slot| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + slot, + Some(T::Pointer::run), + false, + name.as_char_ptr(), + key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `work_struct`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + // + // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that + // the compiler does not complain that the `work` field is unused. + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).work)) } + } +} + +/// Declares that a type contains a [`Work`]. +/// +/// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro +/// like this: +/// +/// ```no_run +/// use kernel::workqueue::{impl_has_work, Work}; +/// +/// struct MyWorkItem { +/// work_field: Work, +/// } +/// +/// impl_has_work! { +/// impl HasWork for MyWorkItem { self.work_field } +/// } +/// ``` +/// +/// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` +/// fields by using a different id for each one. +/// +/// # Safety +/// +/// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be +/// true inverses of each other; that is, they must satisfy the following invariants: +/// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`. +/// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work`. +/// +/// [`impl_has_work!`]: crate::impl_has_work +/// [`raw_get_work`]: HasWork::raw_get_work +/// [`work_container_of`]: HasWork::work_container_of +pub unsafe trait HasWork { + /// Returns a pointer to the [`Work`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work; + + /// Returns a pointer to the struct containing the [`Work`] field. + /// + /// # Safety + /// + /// The pointer must point at a [`Work`] field in a struct of type `Self`. + unsafe fn work_container_of(ptr: *mut Work) -> *mut Self; +} + +/// Used to safely implement the [`HasWork`] trait. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_work, Work}; +/// +/// struct MyStruct<'a, T, const N: usize> { +/// work_field: Work, 17>, +/// f: fn(&'a [T; N]), +/// } +/// +/// impl_has_work! { +/// impl{'a, T, const N: usize} HasWork, 17> +/// for MyStruct<'a, T, N> { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_work { + ($(impl$({$($generics:tt)*})? + HasWork<$work_type:ty $(, $id:tt)?> + for $self:ty + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + } + } + + #[inline] + unsafe fn work_container_of( + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, + ) -> *mut Self { + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + unsafe { $crate::container_of!(ptr, Self, $field) } + } + } + )*}; +} +pub use impl_has_work; + +impl_has_work! { + impl{T} HasWork for ClosureWork { self.work } +} + +/// Links for a delayed work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in +/// a delayed manner. +/// +/// Wraps the kernel's C `struct delayed_work`. +/// +/// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct DelayedWork { + #[pin] + dwork: Opaque, + _inner: PhantomData, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Send for DelayedWork {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Sync for DelayedWork {} + +impl DelayedWork { + /// Creates a new instance of [`DelayedWork`]. + #[inline] + pub fn new( + work_name: &'static CStr, + work_key: Pin<&'static LockClassKey>, + timer_name: &'static CStr, + timer_key: Pin<&'static LockClassKey>, + ) -> impl PinInit + where + T: WorkItem, + { + pin_init!(Self { + dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + core::ptr::addr_of_mut!((*slot).work), + Some(T::Pointer::run), + false, + work_name.as_char_ptr(), + work_key.as_ptr(), + ) + } + + // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because + // the timer is embedded in a `struct delayed_work`, and only ever scheduled via + // the core workqueue code, and configured to run in irqsafe context. + unsafe { + bindings::timer_init_key( + core::ptr::addr_of_mut!((*slot).timer), + Some(bindings::delayed_work_timer_fn), + bindings::TIMER_IRQSAFE, + timer_name.as_char_ptr(), + timer_key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `delayed_work`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let dw: *mut bindings::delayed_work = + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; + // CAST: Work and work_struct have compatible layouts. + wrk.cast() + } +} + +/// Declares that a type contains a [`DelayedWork`]. +/// +/// # Safety +/// +/// The `HasWork` implementation must return a `work_struct` that is stored in the `work` +/// field of a `delayed_work` with the same access rules as the `work_struct`. +pub unsafe trait HasDelayedWork: HasWork {} + +/// Used to safely implement the [`HasDelayedWork`] trait. +/// +/// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] +/// when using this macro. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; +/// +/// struct MyStruct<'a, T, const N: usize> { +/// work_field: DelayedWork, 17>, +/// f: fn(&'a [T; N]), +/// } +/// +/// impl_has_delayed_work! { +/// impl{'a, T, const N: usize} HasDelayedWork, 17> +/// for MyStruct<'a, T, N> { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_delayed_work { + ($(impl$({$($generics:tt)*})? + HasDelayedWork<$work_type:ty $(, $id:tt)?> + for $self:ty + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? + $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} + + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { + #[inline] + unsafe fn raw_get_work( + ptr: *mut Self + ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + }; + + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } + } + + #[inline] + unsafe fn work_container_of( + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, + ) -> *mut Self { + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let delayed_work = unsafe { + $crate::container_of!(ptr, $crate::bindings::delayed_work, work) + }; + + let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = + delayed_work.cast(); + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + unsafe { $crate::container_of!(delayed_work, Self, $field) } + } + } + )*}; +} +pub use impl_has_delayed_work; + +// SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the +// `run` method of this trait as the function pointer because: +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `Arc`. +unsafe impl WorkItemPointer for Arc +where + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let arc = unsafe { Arc::from_raw(ptr) }; + + T::run(arc) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `Arc`, which means that the ref count will be at least 1, +// and we don't drop the `Arc` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for Arc +where + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // Casting between const and mut is not a problem as long as the pointer is a raw pointer. + let ptr = Arc::into_raw(self).cast_mut(); + + // SAFETY: Pointers into an `Arc` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { Arc::from_raw(ptr) }) + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for Arc +where + T: WorkItem, + T: HasDelayedWork, +{ +} + +// SAFETY: TODO. +unsafe impl WorkItemPointer for Pin> +where + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let boxed = unsafe { KBox::from_raw(ptr) }; + // SAFETY: The box was already pinned when it was enqueued. + let pinned = unsafe { Pin::new_unchecked(boxed) }; + + T::run(pinned) + } +} + +// SAFETY: TODO. +unsafe impl RawWorkItem for Pin> +where + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = (); + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily + // remove the `Pin` wrapper. + let boxed = unsafe { Pin::into_inner_unchecked(self) }; + let ptr = KBox::into_raw(boxed); + + // SAFETY: Pointers into a `KBox` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if !queue_work_on(work_ptr) { + // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a + // workqueue. + unsafe { ::core::hint::unreachable_unchecked() } + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for Pin> +where + T: WorkItem, + T: HasDelayedWork, +{ +} + +// SAFETY: Like the `Arc` implementation, the `__enqueue` implementation for +// `ARef` obtains a `work_struct` from the `Work` field using +// `T::raw_get_work`, so the same safety reasoning applies: +// +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `ARef`. +unsafe impl WorkItemPointer for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + + // SAFETY: This computes the pointer that `__enqueue` got from + // `ARef::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + + // SAFETY: The safety contract of `work_container_of` ensures that it + // returns a valid non-null pointer. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: This pointer comes from `ARef::into_raw` and we've been given + // back ownership. + let aref = unsafe { ARef::from_raw(ptr) }; + + T::run(aref) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `ARef`, which means that the ref count will be at least 1, +// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + let ptr = ARef::into_raw(self); + + // SAFETY: Pointers from ARef::into_raw are valid and non-null. + let work_ptr = unsafe { T::raw_get_work(ptr.as_ptr()) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { ARef::from_raw(ptr) }) + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for ARef +where + T: WorkItem, + T: HasDelayedWork, + T: AlwaysRefCounted, +{ +} + +/// Returns the system work queue (`system_wq`). +/// +/// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are +/// users which expect relatively short queue flush time. +/// +/// Callers shouldn't queue work items which can run for too long. +pub fn system() -> &'static Queue { + // SAFETY: `system_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_wq) } +} + +/// Returns the system high-priority work queue (`system_highpri_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which require higher +/// scheduling priority. +pub fn system_highpri() -> &'static Queue { + // SAFETY: `system_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_highpri_wq) } +} + +/// Returns the system work queue for potentially long-running work items (`system_long_wq`). +/// +/// It is similar to the one returned by [`system`] but may host long running work items. Queue +/// flushing might take relatively long. +pub fn system_long() -> &'static Queue { + // SAFETY: `system_long_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_long_wq) } +} + +/// Returns the system unbound work queue (`system_unbound_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as `max_active` limit is not reached and resources are +/// available. +pub fn system_unbound() -> &'static Queue { + // SAFETY: `system_unbound_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_unbound_wq) } +} + +/// Returns the system freezable work queue (`system_freezable_wq`). +/// +/// It is equivalent to the one returned by [`system`] except that it's freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable() -> &'static Queue { + // SAFETY: `system_freezable_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_wq) } +} + +/// Returns the system power-efficient work queue (`system_power_efficient_wq`). +/// +/// It is inclined towards saving power and is converted to "unbound" variants if the +/// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one +/// returned by [`system`]. +pub fn system_power_efficient() -> &'static Queue { + // SAFETY: `system_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } +} + +/// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). +/// +/// It is similar to the one returned by [`system_power_efficient`] except that is freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable_power_efficient() -> &'static Queue { + // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } +} + +/// Returns the system bottom halves work queue (`system_bh_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which +/// need to run from a softirq context. +pub fn system_bh() -> &'static Queue { + // SAFETY: `system_bh_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_wq) } +} + +/// Returns the system bottom halves high-priority work queue (`system_bh_highpri_wq`). +/// +/// It is similar to the one returned by [`system_bh`] but for work items which +/// require higher scheduling priority. +pub fn system_bh_highpri() -> &'static Queue { + // SAFETY: `system_bh_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_highpri_wq) } +} diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs new file mode 100644 index 000000000..987c9c0c2 --- /dev/null +++ b/rust/kernel/xarray.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! XArray abstraction. +//! +//! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) + +use crate::{ + alloc, + bindings, + build_assert::build_assert, + error::{Error, Result}, + ffi::c_void, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, // +}; +use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; +use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; + +/// An array which efficiently maps sparse integer indices to owned objects. +/// +/// This is similar to a [`crate::alloc::kvec::Vec>`], but more efficient when there are +/// holes in the index space, and can be efficiently grown. +/// +/// # Invariants +/// +/// `self.xa` is always an initialized and valid [`bindings::xarray`] whose entries are either +/// `XA_ZERO_ENTRY` or came from `T::into_foreign`. +/// +/// # Examples +/// +/// ```rust +/// use kernel::alloc::KBox; +/// use kernel::xarray::{AllocKind, XArray}; +/// +/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?; +/// +/// let dead = KBox::new(0xdead, GFP_KERNEL)?; +/// let beef = KBox::new(0xbeef, GFP_KERNEL)?; +/// +/// let mut guard = xa.lock(); +/// +/// assert_eq!(guard.get(0), None); +/// +/// assert_eq!(guard.store(0, dead, GFP_KERNEL)?.as_deref(), None); +/// assert_eq!(guard.get(0).copied(), Some(0xdead)); +/// +/// *guard.get_mut(0).unwrap() = 0xffff; +/// assert_eq!(guard.get(0).copied(), Some(0xffff)); +/// +/// assert_eq!(guard.store(0, beef, GFP_KERNEL)?.as_deref().copied(), Some(0xffff)); +/// assert_eq!(guard.get(0).copied(), Some(0xbeef)); +/// +/// guard.remove(0); +/// assert_eq!(guard.get(0), None); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data(PinnedDrop)] +pub struct XArray { + #[pin] + xa: Opaque, + _p: PhantomData, +} + +#[pinned_drop] +impl PinnedDrop for XArray { + fn drop(self: Pin<&mut Self>) { + self.iter().for_each(|ptr| { + let ptr = ptr.as_ptr(); + // SAFETY: `ptr` came from `T::into_foreign`. + // + // INVARIANT: we own the only reference to the array which is being dropped so the + // broken invariant is not observable on function exit. + drop(unsafe { T::from_foreign(ptr) }) + }); + + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_destroy(self.xa.get()) }; + } +} + +/// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior. +pub enum AllocKind { + /// Consider the first element to be at index 0. + Alloc, + /// Consider the first element to be at index 1. + Alloc1, +} + +impl XArray { + /// Creates a new initializer for this type. + pub fn new(kind: AllocKind) -> impl PinInit { + let flags = match kind { + AllocKind::Alloc => bindings::XA_FLAGS_ALLOC, + AllocKind::Alloc1 => bindings::XA_FLAGS_ALLOC1, + }; + pin_init!(Self { + // SAFETY: `xa` is valid while the closure is called. + // + // INVARIANT: `xa` is initialized here to an empty, valid [`bindings::xarray`]. + xa <- Opaque::ffi_init(|xa| unsafe { + bindings::xa_init_flags(xa, flags) + }), + _p: PhantomData, + }) + } + + fn iter(&self) -> impl Iterator> + '_ { + let mut index = 0; + + // SAFETY: `self.xa` is always valid by the type invariant. + iter::once(unsafe { + bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }) + .chain(iter::from_fn(move || { + // SAFETY: `self.xa` is always valid by the type invariant. + Some(unsafe { + bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }) + })) + .map_while(|ptr| NonNull::new(ptr.cast())) + } + + /// Attempts to lock the [`XArray`] for exclusive access. + pub fn try_lock(&self) -> Option> { + // SAFETY: `self.xa` is always valid by the type invariant. + if (unsafe { bindings::xa_trylock(self.xa.get()) } != 0) { + Some(Guard { + xa: self, + _not_send: NotThreadSafe, + }) + } else { + None + } + } + + /// Locks the [`XArray`] for exclusive access. + pub fn lock(&self) -> Guard<'_, T> { + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + Guard { + xa: self, + _not_send: NotThreadSafe, + } + } +} + +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, T: ForeignOwnable> { + xa: &'a XArray, + _not_send: NotThreadSafe, +} + +impl Drop for Guard<'_, T> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::xa_unlock(self.xa.xa.get()) }; + } +} + +/// The error returned by [`store`](Guard::store). +/// +/// Contains the underlying error and the value that was not stored. +pub struct StoreError { + /// The error that occurred. + pub error: Error, + /// The value that was not stored. + pub value: T, +} + +impl From> for Error { + #[inline] + fn from(value: StoreError) -> Self { + value.error + } +} + +impl<'a, T: ForeignOwnable> Guard<'a, T> { + fn load(&self, index: usize, f: F) -> Option + where + F: FnOnce(NonNull) -> U, + { + // SAFETY: `self.xa.xa` is always valid by the type invariant. + let ptr = unsafe { bindings::xa_load(self.xa.xa.get(), index) }; + let ptr = NonNull::new(ptr.cast())?; + Some(f(ptr)) + } + + /// Provides a reference to the element at the given index. + pub fn get(&self, index: usize) -> Option> { + self.load(index, |ptr| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { T::borrow(ptr.as_ptr()) } + }) + } + + /// Provides a mutable reference to the element at the given index. + pub fn get_mut(&mut self, index: usize) -> Option> { + self.load(index, |ptr| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { T::borrow_mut(ptr.as_ptr()) } + }) + } + + /// Removes and returns the element at the given index. + pub fn remove(&mut self, index: usize) -> Option { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock. + let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast(); + // SAFETY: + // - `ptr` is either NULL or came from `T::into_foreign`. + // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`] + // borrowed from `self` have ended. + unsafe { T::try_from_foreign(ptr) } + } + + /// Stores an element at the given index. + /// + /// May drop the lock if needed to allocate memory, and then reacquire it afterwards. + /// + /// On success, returns the element which was previously at the given index. + /// + /// On failure, returns the element which was attempted to be stored. + pub fn store( + &mut self, + index: usize, + value: T, + gfp: alloc::Flags, + ) -> Result, StoreError> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let new = value.into_foreign(); + + let old = { + let new = new.cast(); + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock. + // + // INVARIANT: `new` came from `T::into_foreign`. + unsafe { bindings::__xa_store(self.xa.xa.get(), index, new, gfp.as_raw()) } + }; + + // SAFETY: `__xa_store` returns the old entry at this index on success or `xa_err` if an + // error happened. + let errno = unsafe { bindings::xa_err(old) }; + if errno != 0 { + // SAFETY: `new` came from `T::into_foreign` and `__xa_store` does not take + // ownership of the value on error. + let value = unsafe { T::from_foreign(new) }; + Err(StoreError { + value, + error: Error::from_errno(errno), + }) + } else { + let old = old.cast(); + // SAFETY: `ptr` is either NULL or came from `T::into_foreign`. + // + // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray + // API; such entries present as `NULL`. + Ok(unsafe { T::try_from_foreign(old) }) + } + } +} + +// SAFETY: `XArray` has no shared mutable state so it is `Send` iff `T` is `Send`. +unsafe impl Send for XArray {} + +// SAFETY: `XArray` serialises the interior mutability it provides so it is `Sync` iff `T` is +// `Send`. +unsafe impl Sync for XArray {} diff --git a/rust/macros/concat_idents.rs b/rust/macros/concat_idents.rs new file mode 100644 index 000000000..47b6add37 --- /dev/null +++ b/rust/macros/concat_idents.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::{ + Ident, + TokenStream, + TokenTree, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + Result, + Token, // +}; + +pub(crate) struct Input { + a: Ident, + _comma: Token![,], + b: Ident, +} + +impl Parse for Input { + fn parse(input: ParseStream<'_>) -> Result { + Ok(Self { + a: input.parse()?, + _comma: input.parse()?, + b: input.parse()?, + }) + } +} + +pub(crate) fn concat_idents(Input { a, b, .. }: Input) -> TokenStream { + let res = Ident::new(&format!("{a}{b}"), b.span()); + TokenStream::from_iter([TokenTree::Ident(res)]) +} diff --git a/rust/macros/export.rs b/rust/macros/export.rs new file mode 100644 index 000000000..6d53521f6 --- /dev/null +++ b/rust/macros/export.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::TokenStream; +use quote::quote; + +/// Please see [`crate::export`] for documentation. +pub(crate) fn export(f: syn::ItemFn) -> TokenStream { + let name = &f.sig.ident; + + quote! { + // This verifies that the function has the same signature as the declaration generated by + // bindgen. It makes use of the fact that all branches of an if/else must have the same + // type. + const _: () = { + if true { + ::kernel::bindings::#name + } else { + #name + }; + }; + + #[no_mangle] + #f + } +} diff --git a/rust/macros/fmt.rs b/rust/macros/fmt.rs new file mode 100644 index 000000000..ce6c72493 --- /dev/null +++ b/rust/macros/fmt.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::collections::BTreeSet; + +use proc_macro2::{Ident, TokenStream, TokenTree}; +use quote::quote_spanned; + +/// Please see [`crate::fmt`] for documentation. +pub(crate) fn fmt(input: TokenStream) -> TokenStream { + let mut input = input.into_iter(); + + let first_opt = input.next(); + let first_owned_str; + let mut names = BTreeSet::new(); + let first_span = { + let Some((mut first_str, first_span)) = (match first_opt.as_ref() { + Some(TokenTree::Literal(first_lit)) => { + first_owned_str = first_lit.to_string(); + Some(first_owned_str.as_str()).and_then(|first| { + let first = first.strip_prefix('"')?; + let first = first.strip_suffix('"')?; + Some((first, first_lit.span())) + }) + } + _ => None, + }) else { + return first_opt.into_iter().chain(input).collect(); + }; + + // Parse `identifier`s from the format string. + // + // See https://doc.rust-lang.org/std/fmt/index.html#syntax. + while let Some((_, rest)) = first_str.split_once('{') { + first_str = rest; + if let Some(rest) = first_str.strip_prefix('{') { + first_str = rest; + continue; + } + if let Some((name, rest)) = first_str.split_once('}') { + first_str = rest; + let name = name.split_once(':').map_or(name, |(name, _)| name); + if !name.is_empty() && !name.chars().all(|c| c.is_ascii_digit()) { + names.insert(name); + } + } + } + first_span + }; + + let adapter = quote_spanned!(first_span => ::kernel::fmt::Adapter); + + let mut args = TokenStream::from_iter(first_opt); + { + let mut flush = |args: &mut TokenStream, current: &mut TokenStream| { + let current = std::mem::take(current); + if !current.is_empty() { + let (lhs, rhs) = (|| { + let mut current = current.into_iter(); + let mut acc = TokenStream::new(); + while let Some(tt) = current.next() { + // Split on `=` only once to handle cases like `a = b = c`. + if matches!(&tt, TokenTree::Punct(p) if p.as_char() == '=') { + names.remove(acc.to_string().as_str()); + // Include the `=` itself to keep the handling below uniform. + acc.extend([tt]); + return (Some(acc), current.collect::()); + } + acc.extend([tt]); + } + (None, acc) + })(); + args.extend(quote_spanned!(first_span => #lhs #adapter(&(#rhs)))); + } + }; + + let mut current = TokenStream::new(); + for tt in input { + match &tt { + TokenTree::Punct(p) if p.as_char() == ',' => { + flush(&mut args, &mut current); + &mut args + } + _ => &mut current, + } + .extend([tt]); + } + flush(&mut args, &mut current); + } + + for name in names { + let name = Ident::new(name, first_span); + args.extend(quote_spanned!(first_span => , #name = #adapter(&#name))); + } + + quote_spanned!(first_span => ::core::format_args!(#args)) +} diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs new file mode 100644 index 000000000..4372cbad3 --- /dev/null +++ b/rust/macros/for_lt.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{ + Span, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + visit::Visit, + visit_mut::VisitMut, + Lifetime, + Result, + Token, + Type, // +}; + +pub(crate) enum HigherRankedType { + Explicit { + _for_token: Token![for], + _lt_token: Token![<], + lifetime: Lifetime, + _gt_token: Token![>], + ty: Type, + }, + Implicit { + ty: Type, + }, +} + +impl Parse for HigherRankedType { + fn parse(input: ParseStream<'_>) -> Result { + if input.peek(Token![for]) { + Ok(Self::Explicit { + _for_token: input.parse()?, + _lt_token: input.parse()?, + lifetime: input.parse()?, + _gt_token: input.parse()?, + ty: input.parse()?, + }) + } else { + Ok(Self::Implicit { ty: input.parse()? }) + } + } +} + +trait TypeExt { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type; + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type; + fn has_lifetime(&self, lt: &Lifetime) -> bool; +} + +impl TypeExt for Type { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type { + struct ElidedLifetimeExpander<'a>(&'a Lifetime); + + impl VisitMut for ElidedLifetimeExpander<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + // Expand explicit `'_` + if lifetime.ident == "_" { + *lifetime = self.0.clone(); + } + } + + fn visit_type_reference_mut(&mut self, reference: &mut syn::TypeReference) { + syn::visit_mut::visit_type_reference_mut(self, reference); + + if reference.lifetime.is_none() { + reference.lifetime = Some(self.0.clone()); + } + } + } + + let mut ret = self.clone(); + ElidedLifetimeExpander(explicit_lt).visit_type_mut(&mut ret); + ret + } + + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type { + struct LifetimeReplacer<'a>(&'a Lifetime, &'a Lifetime); + + impl VisitMut for LifetimeReplacer<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + if lifetime.ident == self.0.ident { + *lifetime = self.1.clone(); + } + } + } + + let mut ret = self.clone(); + LifetimeReplacer(src, dst).visit_type_mut(&mut ret); + ret + } + + fn has_lifetime(&self, lt: &Lifetime) -> bool { + struct HasLifetime<'a>(&'a Lifetime, bool); + + impl Visit<'_> for HasLifetime<'_> { + fn visit_lifetime(&mut self, lifetime: &Lifetime) { + if lifetime.ident == self.0.ident { + self.1 = true; + } + } + + // Macro invocations are opaque; conservatively assume they may + // reference the lifetime. + fn visit_macro(&mut self, _: &syn::Macro) { + self.1 = true; + } + } + + let mut visitor = HasLifetime(lt, false); + visitor.visit_type(self); + visitor.1 + } +} + +struct Prover<'a>(&'a Lifetime, Vec<&'a Type>); + +impl<'a> Prover<'a> { + /// Prove that `ty` is covariant over `'lt`. + /// + /// This also needs to prove that it'll be wellformed for any instance of `'lt`. + /// It can be assumed that `ty` will be wellformed if `'lt` is substituted to `'static`. + fn prove(&mut self, ty: &'a Type) { + match ty { + Type::Paren(ty) => self.prove(&ty.elem), + Type::Group(ty) => self.prove(&ty.elem), + + // No lifetime involved + Type::Never(_) => {} + + // `[T; N]` and `[T]` is covariant over `T`. + Type::Array(ty) => self.prove(&ty.elem), + Type::Slice(ty) => self.prove(&ty.elem), + + Type::Tuple(ty) => { + for elem in &ty.elems { + self.prove(elem); + } + } + + // `*const T` is covariant over `T` + Type::Ptr(ty) if ty.const_token.is_some() => self.prove(&ty.elem), + + // `&T` is covariant over `T` and lifetime. + // + // Note that if we encounter `&'other_lt T`, then we still need to make sure the type + // is wellformed if `T` involves `&'lt`, so we defer to the compiler. + // + // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the + // presence of the type implies `'a: 'static` but this is unsound. + Type::Reference(ty) + if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => + { + self.prove(&ty.elem) + } + + // `&[mut] T` is covariant over lifetime. + // In case we have `&[mut] NoLifetime`, we don't need to do additional checks. + Type::Reference(ty) if !ty.elem.has_lifetime(self.0) => (), + + // No mention of lifetime at all, no need to perform compiler check. + ty if !ty.has_lifetime(self.0) => (), + + // Otherwise, we need to emit checks so that compiler can determine if the types are + // actually covariant. + ty => self.1.push(ty), + } + } +} + +/// Shared implementation for both `ForLt!` and `CovariantForLt!`. +/// +/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime +/// instances (workaround for ). `CovariantForLt!` +/// additionally emits covariance proof functions and sets `N = 1`. +fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream { + let (ty, lifetime) = match input { + HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), + HigherRankedType::Implicit { ty } => { + // If there's no explicit `for<'a>` binder, inject a synthetic `'__elided` lifetime + // and expand elided sites. + let lifetime = Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("__elided", span = Span::mixed_site()), + }; + (ty.expand_elided_lifetime(&lifetime), lifetime) + } + }; + + let mut prover = Prover(&lifetime, Vec::new()); + prover.prove(&ty); + + let mut proof = Vec::new(); + + // Emit proofs for every type that requires additional compiler help in proving covariance. + for (idx, required_proof) in prover.1.into_iter().enumerate() { + // Insert a proof that the type is well-formed. + // + // This is intended to workaround a Rust compiler soundness bug related to HRTB. + // https://github.com/rust-lang/rust/issues/152489 + // + // This needs to be a struct instead of fn to avoid the implied WF bounds. + let wf_proof_name = format_ident!("ProveWf{idx}"); + proof.push(quote!( + struct #wf_proof_name<#lifetime>( + ::core::marker::PhantomData<&#lifetime ()>, #required_proof + ); + )); + + // Insert a proof that the type is covariant. + if prove_covariance { + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } + } + + // Make sure that the type is wellformed when substituting lifetime with `'static`. + // + // Currently the Rust compiler doesn't check this, see the above `ProveWf` documentation. + // + // We prefer to use this way of proving WF-ness as it can work when generics are involved. + let ty_static = ty.replace_lifetime( + &lifetime, + &Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("static"), + }, + ); + + let n: usize = prove_covariance.into(); + + quote!( + ::kernel::types::for_lt::UnsafeForLtImpl::< + dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, + #ty_static, + { + #(#proof)* + + #n + } + > + ) +} + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, false) +} + +pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, true) +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs new file mode 100644 index 000000000..d18fbf4da --- /dev/null +++ b/rust/macros/helpers.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + Attribute, + Error, + LitStr, + Result, // +}; + +/// A string literal that is required to have ASCII value only. +pub(crate) struct AsciiLitStr(LitStr); + +impl Parse for AsciiLitStr { + fn parse(input: ParseStream<'_>) -> Result { + let s: LitStr = input.parse()?; + if !s.value().is_ascii() { + return Err(Error::new_spanned(s, "expected ASCII-only string literal")); + } + Ok(Self(s)) + } +} + +impl ToTokens for AsciiLitStr { + fn to_tokens(&self, ts: &mut TokenStream) { + self.0.to_tokens(ts); + } +} + +impl AsciiLitStr { + pub(crate) fn value(&self) -> String { + self.0.value() + } +} + +pub(crate) fn file() -> String { + #[cfg(not(CONFIG_RUSTC_HAS_SPAN_FILE))] + { + proc_macro::Span::call_site() + .source_file() + .path() + .to_string_lossy() + .into_owned() + } + + #[cfg(CONFIG_RUSTC_HAS_SPAN_FILE)] + { + proc_macro::Span::call_site().file() + } +} + +/// Obtain all `#[cfg]` attributes. +pub(crate) fn gather_cfg_attrs(attr: &[Attribute]) -> impl Iterator + '_ { + attr.iter().filter(|a| a.path().is_ident("cfg")) +} diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs new file mode 100644 index 000000000..ae20ed676 --- /dev/null +++ b/rust/macros/kunit.rs @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Procedural macro to run KUnit tests using a user-space like syntax. +//! +//! Copyright (c) 2023 José Expósito + +use std::ffi::CString; + +use proc_macro2::TokenStream; +use quote::{ + format_ident, + quote, + ToTokens, // +}; +use syn::{ + parse_quote, + Error, + Ident, + Item, + ItemMod, + LitCStr, + Result, // +}; + +pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result { + if test_suite.to_string().len() > 255 { + return Err(Error::new_spanned( + test_suite, + "test suite names cannot exceed the maximum length of 255 bytes", + )); + } + + // We cannot handle modules that defer to another file (e.g. `mod foo;`). + let Some((module_brace, module_items)) = module.content.take() else { + Err(Error::new_spanned( + module, + "`#[kunit_tests(test_name)]` attribute should only be applied to inline modules", + ))? + }; + + // Make the entire module gated behind `CONFIG_KUNIT`. + module + .attrs + .insert(0, parse_quote!(#[cfg(CONFIG_KUNIT="y")])); + + let mut processed_items = Vec::new(); + let mut test_cases = Vec::new(); + + // Generate the test KUnit test suite and a test case for each `#[test]`. + // + // The code generated for the following test module: + // + // ``` + // #[kunit_tests(kunit_test_suit_name)] + // mod tests { + // #[test] + // fn foo() { + // assert_eq!(1, 1); + // } + // + // #[test] + // fn bar() { + // assert_eq!(2, 2); + // } + // } + // ``` + // + // Looks like: + // + // ``` + // unsafe extern "C" fn kunit_rust_wrapper_foo(_test: *mut ::kernel::bindings::kunit) { foo(); } + // unsafe extern "C" fn kunit_rust_wrapper_bar(_test: *mut ::kernel::bindings::kunit) { bar(); } + // + // static mut TEST_CASES: [::kernel::bindings::kunit_case; 3] = [ + // ::kernel::kunit::kunit_case(c"foo", kunit_rust_wrapper_foo), + // ::kernel::kunit::kunit_case(c"bar", kunit_rust_wrapper_bar), + // ::pin_init::zeroed(), + // ]; + // + // ::kernel::kunit_unsafe_test_suite!(kunit_test_suit_name, TEST_CASES); + // ``` + // + // Non-function items (e.g. imports) are preserved. + for item in module_items { + let Item::Fn(mut f) = item else { + processed_items.push(item); + continue; + }; + + if f.attrs + .extract_if(.., |attr| attr.path().is_ident("test")) + .count() + == 0 + { + processed_items.push(Item::Fn(f)); + continue; + } + + let test = f.sig.ident.clone(); + + // Retrieve `#[cfg]` applied on the function which needs to be present on derived items too. + let cfg_attrs: Vec<_> = f + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .cloned() + .collect(); + + // Before the test, override usual `assert!` and `assert_eq!` macros with ones that call + // KUnit instead. + let test_str = test.to_string(); + let path = CString::new(crate::helpers::file()).expect("file path cannot contain NUL"); + processed_items.push(parse_quote! { + #[allow(unused)] + macro_rules! assert { + ($cond:expr $(,)?) => {{ + kernel::kunit_assert!(#test_str, #path, 0, $cond); + }} + } + }); + processed_items.push(parse_quote! { + #[allow(unused)] + macro_rules! assert_eq { + ($left:expr, $right:expr $(,)?) => {{ + kernel::kunit_assert_eq!(#test_str, #path, 0, $left, $right); + }} + } + }); + + // Add back the test item. + processed_items.push(Item::Fn(f)); + + let kunit_wrapper_fn_name = format_ident!("kunit_rust_wrapper_{test}"); + let test_cstr = LitCStr::new( + &CString::new(test_str.as_str()).expect("identifier cannot contain NUL"), + test.span(), + ); + processed_items.push(parse_quote! { + unsafe extern "C" fn #kunit_wrapper_fn_name(_test: *mut ::kernel::bindings::kunit) { + (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SKIPPED; + + // Append any `cfg` attributes the user might have written on their tests so we + // don't attempt to call them when they are `cfg`'d out. An extra `use` is used + // here to reduce the length of the assert message. + #(#cfg_attrs)* + { + (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SUCCESS; + use ::kernel::kunit::is_test_result_ok; + assert!(is_test_result_ok(#test())); + } + } + }); + + test_cases.push(quote!( + ::kernel::kunit::kunit_case(#test_cstr, #kunit_wrapper_fn_name) + )); + } + + let num_tests_plus_1 = test_cases.len() + 1; + processed_items.push(parse_quote! { + static mut TEST_CASES: [::kernel::bindings::kunit_case; #num_tests_plus_1] = [ + #(#test_cases,)* + ::pin_init::zeroed(), + ]; + }); + processed_items.push(parse_quote! { + ::kernel::kunit_unsafe_test_suite!(#test_suite, TEST_CASES); + }); + + module.content = Some((module_brace, processed_items)); + Ok(module.to_token_stream()) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs new file mode 100644 index 000000000..24f96feae --- /dev/null +++ b/rust/macros/lib.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Crate for all kernel procedural macros. + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +// Stable since Rust 1.87.0. +#![feature(extract_if)] +// +// Stable since Rust 1.88.0 under a different name, `proc_macro_span_file`, +// which was added in Rust 1.88.0. This is why `cfg_attr` is used here, i.e. +// to avoid depending on the full `proc_macro_span` on Rust >= 1.88.0. +#![cfg_attr(not(CONFIG_RUSTC_HAS_SPAN_FILE), feature(proc_macro_span))] + +mod concat_idents; +mod export; +mod fmt; +mod for_lt; +mod helpers; +mod kunit; +mod module; +mod paste; +mod vtable; + +use proc_macro::TokenStream; + +use syn::parse_macro_input; + +/// Declares a kernel module. +/// +/// The `type` argument should be a type which implements the [`Module`] +/// trait. Also accepts various forms of kernel metadata. +/// +/// The `params` field describe module parameters. Each entry has the form +/// +/// ```ignore +/// parameter_name: type { +/// default: default_value, +/// description: "Description", +/// } +/// ``` +/// +/// `type` may be one of +/// +/// - [`i8`] +/// - [`u8`] +/// - [`i8`] +/// - [`u8`] +/// - [`i16`] +/// - [`u16`] +/// - [`i32`] +/// - [`u32`] +/// - [`i64`] +/// - [`u64`] +/// - [`isize`] +/// - [`usize`] +/// - [`bool`] +/// +/// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) +/// +/// [`Module`]: ../kernel/trait.Module.html +/// +/// # Examples +/// +/// ```ignore +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyModule, +/// name: "my_kernel_module", +/// authors: ["Rust for Linux Contributors"], +/// description: "My very own kernel module!", +/// license: "GPL", +/// alias: ["alternate_module_name"], +/// params: { +/// my_parameter: i64 { +/// default: 1, +/// description: "This parameter has a default of 1", +/// }, +/// }, +/// } +/// +/// struct MyModule(i32); +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// let foo: i32 = 42; +/// pr_info!("I contain: {}\n", foo); +/// pr_info!("i32 param is: {}\n", module_parameters::my_parameter.read()); +/// Ok(Self(foo)) +/// } +/// } +/// # fn main() {} +/// ``` +/// +/// ## Firmware +/// +/// The following example shows how to declare a kernel module that needs +/// to load binary firmware files. You need to specify the file names of +/// the firmware in the `firmware` field. The information is embedded +/// in the `modinfo` section of the kernel module. For example, a tool to +/// build an initramfs uses this information to put the firmware files into +/// the initramfs image. +/// +/// ``` +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyDeviceDriverModule, +/// name: "my_device_driver_module", +/// authors: ["Rust for Linux Contributors"], +/// description: "My device driver requires firmware", +/// license: "GPL", +/// firmware: ["my_device_firmware1.bin", "my_device_firmware2.bin"], +/// } +/// +/// struct MyDeviceDriverModule; +/// +/// impl kernel::Module for MyDeviceDriverModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// Ok(Self) +/// } +/// } +/// # fn main() {} +/// ``` +/// +/// # Supported argument types +/// - `type`: type which implements the [`Module`] trait (required). +/// - `name`: ASCII string literal of the name of the kernel module (required). +/// - `authors`: array of ASCII string literals of the authors of the kernel module. +/// - `description`: string literal of the description of the kernel module. +/// - `license`: ASCII string literal of the license of the kernel module (required). +/// - `alias`: array of ASCII string literals of the alias names of the kernel module. +/// - `firmware`: array of ASCII string literals of the firmware files of +/// the kernel module. +#[proc_macro] +pub fn module(input: TokenStream) -> TokenStream { + module::module(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Declares or implements a vtable trait. +/// +/// Linux's use of pure vtables is very close to Rust traits, but they differ +/// in how unimplemented functions are represented. In Rust, traits can provide +/// default implementation for all non-required methods (and the default +/// implementation could just return `Error::EINVAL`); Linux typically use C +/// `NULL` pointers to represent these functions. +/// +/// This attribute closes that gap. A trait can be annotated with the +/// `#[vtable]` attribute. Implementers of the trait will then also have to +/// annotate the trait with `#[vtable]`. This attribute generates a `HAS_*` +/// associated constant bool for each method in the trait that is set to true if +/// the implementer has overridden the associated method. +/// +/// For a trait method to be optional, it must have a default implementation. +/// This is also the case for traits annotated with `#[vtable]`, but in this +/// case the default implementation will never be executed. The reason for this +/// is that the functions will be called through function pointers installed in +/// C side vtables. When an optional method is not implemented on a `#[vtable]` +/// trait, a `NULL` entry is installed in the vtable. Thus the default +/// implementation is never called. Since these traits are not designed to be +/// used on the Rust side, it should not be possible to call the default +/// implementation. This is done to ensure that we call the vtable methods +/// through the C vtable, and not through the Rust vtable. Therefore, the +/// default implementation should call `build_error!`, which prevents +/// calls to this function at compile time: +/// +/// ```compile_fail +/// # // Intentionally missing `use`s to simplify `rusttest`. +/// build_error!(VTABLE_DEFAULT_ERROR) +/// ``` +/// +/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`]. +/// +/// This macro should not be used when all functions are required. +/// +/// Additionally, this macro automatically handles the `OwnerModule` +/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;` +/// is added as a required associated type if not already defined; on the +/// impl side, `type OwnerModule = LocalModule;` is automatically inserted +/// if not explicitly defined. +/// +/// # Examples +/// +/// ``` +/// use kernel::error::VTABLE_DEFAULT_ERROR; +/// use kernel::prelude::*; +/// +/// # struct LocalModule; +/// # impl kernel::ModuleMetadata for LocalModule { +/// # const NAME: &'static kernel::str::CStr = c"vtable_doctest"; +/// # +/// # // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`. +/// # const THIS_MODULE: kernel::ThisModule = unsafe { +/// # kernel::ThisModule::from_ptr(core::ptr::null_mut()) +/// # }; +/// # } +/// # +/// # fn main() { +/// // Declares a `#[vtable]` trait +/// #[vtable] +/// pub trait Operations: Send + Sync + Sized { +/// fn foo(&self) -> Result<()> { +/// build_error!(VTABLE_DEFAULT_ERROR) +/// } +/// +/// fn bar(&self) -> Result<()> { +/// build_error!(VTABLE_DEFAULT_ERROR) +/// } +/// } +/// +/// struct Foo; +/// +/// // Implements the `#[vtable]` trait +/// #[vtable] +/// impl Operations for Foo { +/// fn foo(&self) -> Result<()> { +/// # Err(EINVAL) +/// // ... +/// } +/// } +/// +/// assert_eq!(::HAS_FOO, true); +/// assert_eq!(::HAS_BAR, false); +/// # } +/// ``` +/// +/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html +#[proc_macro_attribute] +pub fn vtable(attr: TokenStream, input: TokenStream) -> TokenStream { + parse_macro_input!(attr as syn::parse::Nothing); + vtable::vtable(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Export a function so that C code can call it via a header file. +/// +/// Functions exported using this macro can be called from C code using the declaration in the +/// appropriate header file. It should only be used in cases where C calls the function through a +/// header file; cases where C calls into Rust via a function pointer in a vtable (such as +/// `file_operations`) should not use this macro. +/// +/// This macro has the following effect: +/// +/// * Disables name mangling for this function. +/// * Verifies at compile-time that the function signature matches the declaration in the header +/// file. +/// +/// You must declare the signature of the Rust function in a header file that is included by +/// `rust/bindings/bindings_helper.h`. +/// +/// This macro is *not* the same as the C macros `EXPORT_SYMBOL_*`. All Rust symbols are currently +/// automatically exported with `EXPORT_SYMBOL_GPL`. +#[proc_macro_attribute] +pub fn export(attr: TokenStream, input: TokenStream) -> TokenStream { + parse_macro_input!(attr as syn::parse::Nothing); + export::export(parse_macro_input!(input)).into() +} + +/// Like [`core::format_args!`], but automatically wraps arguments in [`kernel::fmt::Adapter`]. +/// +/// This macro allows generating `fmt::Arguments` while ensuring that each argument is wrapped with +/// `::kernel::fmt::Adapter`, which customizes formatting behavior for kernel logging. +/// +/// Named arguments used in the format string (e.g. `{foo}`) are detected and resolved from local +/// bindings. All positional and named arguments are automatically wrapped. +/// +/// This macro is an implementation detail of other kernel logging macros like [`pr_info!`] and +/// should not typically be used directly. +/// +/// [`kernel::fmt::Adapter`]: ../kernel/fmt/struct.Adapter.html +/// [`pr_info!`]: ../kernel/macro.pr_info.html +#[proc_macro] +pub fn fmt(input: TokenStream) -> TokenStream { + fmt::fmt(input.into()).into() +} + +/// Concatenate two identifiers. +/// +/// This is useful in macros that need to declare or reference items with names +/// starting with a fixed prefix and ending in a user specified name. The resulting +/// identifier has the span of the second argument. +/// +/// # Examples +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// use kernel::macros::concat_idents; +/// +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// $(pub(crate) const $newname: u32 = concat_idents!($prefix, $newname);)+ +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK); +/// ``` +#[proc_macro] +pub fn concat_idents(input: TokenStream) -> TokenStream { + concat_idents::concat_idents(parse_macro_input!(input)).into() +} + +/// Paste identifiers together. +/// +/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a +/// single identifier. +/// +/// This is similar to the [`paste`] crate, but with pasting feature limited to identifiers and +/// literals (lifetimes and documentation strings are not supported). There is a difference in +/// supported modifiers as well. +/// +/// # Examples +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// ::kernel::macros::paste! { +/// $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Modifiers +/// +/// For each identifier, it is possible to attach one or multiple modifiers to +/// it. +/// +/// Currently supported modifiers are: +/// * `span`: change the span of concatenated identifier to the span of the specified token. By +/// default the span of the `[< >]` group is used. +/// * `lower`: change the identifier to lower case. +/// * `upper`: change the identifier to upper case. +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// ::kernel::macros::paste! { +/// $(pub(crate) const fn [<$newname:lower:span>]() -> u32 { [<$prefix $newname:span>] })+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(br_ok(), binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Literals +/// +/// Literals can also be concatenated with other identifiers: +/// +/// ``` +/// macro_rules! create_numbered_fn { +/// ($name:literal, $val:literal) => { +/// ::kernel::macros::paste! { +/// fn []() -> u32 { $val } +/// } +/// }; +/// } +/// +/// create_numbered_fn!("foo", 100); +/// +/// assert_eq!(some_foo_fn100(), 100) +/// ``` +/// +/// [`paste`]: https://docs.rs/paste/ +#[proc_macro] +pub fn paste(input: TokenStream) -> TokenStream { + let mut tokens = proc_macro2::TokenStream::from(input).into_iter().collect(); + paste::expand(&mut tokens); + tokens + .into_iter() + .collect::() + .into() +} + +/// Registers a KUnit test suite and its test cases using a user-space like syntax. +/// +/// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module +/// is ignored. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::prelude::*; +/// #[kunit_tests(kunit_test_suit_name)] +/// mod tests { +/// #[test] +/// fn foo() { +/// assert_eq!(1, 1); +/// } +/// +/// #[test] +/// fn bar() { +/// assert_eq!(2, 2); +/// } +/// } +/// ``` +#[proc_macro_attribute] +pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { + kunit::kunit_tests(parse_macro_input!(attr), parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Obtain a type that implements [`ForLt`] for the given higher-ranked type. +/// +/// Please refer to the documentation of the [`ForLt`] trait. +/// +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn ForLt(input: TokenStream) -> TokenStream { + for_lt::for_lt(parse_macro_input!(input)).into() +} + +/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked +/// type. +/// +/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime, +/// providing a safe [`CovariantForLt::cast_ref`] method. +/// +/// Please refer to the documentation of the [`CovariantForLt`] trait. +/// +/// [`CovariantForLt`]: trait.CovariantForLt.html +/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn CovariantForLt(input: TokenStream) -> TokenStream { + for_lt::covariant_for_lt(parse_macro_input!(input)).into() +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs new file mode 100644 index 000000000..bc7027f8d --- /dev/null +++ b/rust/macros/module.rs @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::ffi::CString; + +use proc_macro2::{ + Literal, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + braced, + bracketed, + ext::IdentExt, + parse::{ + Parse, + ParseStream, // + }, + parse_quote, + punctuated::Punctuated, + Error, + Expr, + Ident, + LitStr, + Path, + Result, + Token, + Type, // +}; + +use crate::helpers::*; + +struct ModInfoBuilder<'a> { + module: &'a str, + counter: usize, + ts: TokenStream, + param_ts: TokenStream, +} + +impl<'a> ModInfoBuilder<'a> { + fn new(module: &'a str) -> Self { + ModInfoBuilder { + module, + counter: 0, + ts: TokenStream::new(), + param_ts: TokenStream::new(), + } + } + + fn emit_base(&mut self, field: &str, content: &str, builtin: bool, param: bool) { + let string = if builtin { + // Built-in modules prefix their modinfo strings by `module.`. + format!("{module}.{field}={content}\0", module = self.module) + } else { + // Loadable modules' modinfo strings go as-is. + format!("{field}={content}\0") + }; + let length = string.len(); + let string = Literal::byte_string(string.as_bytes()); + let cfg = if builtin { + quote!(#[cfg(not(MODULE))]) + } else { + quote!(#[cfg(MODULE)]) + }; + + let counter = format_ident!( + "__{module}_{counter}", + module = self.module.to_uppercase(), + counter = self.counter + ); + let item = quote! { + #cfg + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + pub static #counter: [u8; #length] = *#string; + }; + + if param { + self.param_ts.extend(item); + } else { + self.ts.extend(item); + } + + self.counter += 1; + } + + fn emit_only_builtin(&mut self, field: &str, content: &str, param: bool) { + self.emit_base(field, content, true, param) + } + + fn emit_only_loadable(&mut self, field: &str, content: &str, param: bool) { + self.emit_base(field, content, false, param) + } + + fn emit(&mut self, field: &str, content: &str) { + self.emit_internal(field, content, false); + } + + fn emit_internal(&mut self, field: &str, content: &str, param: bool) { + self.emit_only_builtin(field, content, param); + self.emit_only_loadable(field, content, param); + } + + fn emit_param(&mut self, field: &str, param: &str, content: &str) { + let content = format!("{param}:{content}"); + self.emit_internal(field, &content, true); + } + + fn emit_params(&mut self, info: &ModuleInfo) { + let Some(params) = &info.params else { + return; + }; + + for param in params { + let param_name_str = param.name.to_string(); + let param_type_str = param.ptype.to_string(); + + let ops = param_ops_path(¶m_type_str); + + // Note: The spelling of these fields is dictated by the user space + // tool `modinfo`. + self.emit_param("parmtype", ¶m_name_str, ¶m_type_str); + self.emit_param("parm", ¶m_name_str, ¶m.description.value()); + + let static_name = format_ident!("__{}_{}_struct", self.module, param.name); + let param_name_cstr = + CString::new(param_name_str).expect("name contains NUL-terminator"); + let param_name_cstr_with_module = + CString::new(format!("{}.{}", self.module, param.name)) + .expect("name contains NUL-terminator"); + + let param_name = ¶m.name; + let param_type = ¶m.ptype; + let param_default = ¶m.default; + + self.param_ts.extend(quote! { + #[allow(non_upper_case_globals)] + pub(crate) static #param_name: + ::kernel::module_param::ModuleParamAccess<#param_type> = + ::kernel::module_param::ModuleParamAccess::new(#param_default); + + const _: () = { + #[allow(non_upper_case_globals)] + #[link_section = "__param"] + #[used(compiler)] + static #static_name: + ::kernel::module_param::KernelParam = + ::kernel::module_param::KernelParam::new( + ::kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context( + if ::core::cfg!(MODULE) { + #param_name_cstr + } else { + #param_name_cstr_with_module + } + ), + // SAFETY: `__this_module` is constructed by the kernel at load + // time and will not be freed until the module is unloaded. + #[cfg(MODULE)] + mod_: unsafe { + core::ptr::from_ref(&::kernel::bindings::__this_module) + .cast_mut() + }, + #[cfg(not(MODULE))] + mod_: ::core::ptr::null_mut(), + ops: core::ptr::from_ref(&#ops), + perm: 0, // Will not appear in sysfs + level: -1, + flags: 0, + __bindgen_anon_1: ::kernel::bindings::kernel_param__bindgen_ty_1 { + arg: #param_name.as_void_ptr() + }, + } + ); + }; + }); + } + } +} + +fn param_ops_path(param_type: &str) -> Path { + match param_type { + "i8" => parse_quote!(::kernel::module_param::PARAM_OPS_I8), + "u8" => parse_quote!(::kernel::module_param::PARAM_OPS_U8), + "i16" => parse_quote!(::kernel::module_param::PARAM_OPS_I16), + "u16" => parse_quote!(::kernel::module_param::PARAM_OPS_U16), + "i32" => parse_quote!(::kernel::module_param::PARAM_OPS_I32), + "u32" => parse_quote!(::kernel::module_param::PARAM_OPS_U32), + "i64" => parse_quote!(::kernel::module_param::PARAM_OPS_I64), + "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64), + "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE), + "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE), + "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL), + t => panic!("Unsupported parameter type {}", t), + } +} + +/// Parse fields that are required to use a specific order. +/// +/// As fields must follow a specific order, we *could* just parse fields one by one by peeking. +/// However the error message generated when implementing that way is not very friendly. +/// +/// So instead we parse fields in an arbitrary order, but only enforce the ordering after parsing, +/// and if the wrong order is used, the proper order is communicated to the user with error message. +/// +/// Usage looks like this: +/// ```ignore +/// parse_ordered_fields! { +/// from input; +/// +/// // This will extract "foo: " into a variable named "foo". +/// // The variable will have type `Option<_>`. +/// foo => , +/// +/// // If you need the variable name to be different than the key name. +/// // This extracts "baz: " into a variable named "bar". +/// // You might want this if "baz" is a keyword. +/// baz as bar => , +/// +/// // You can mark a key as required, and the variable will no longer be `Option`. +/// // foobar will be of type `Expr` instead of `Option`. +/// foobar [required] => input.parse::()?, +/// } +/// ``` +macro_rules! parse_ordered_fields { + (@gen + [$input:expr] + [$([$name:ident; $key:ident; $parser:expr])*] + [$([$req_name:ident; $req_key:ident])*] + ) => { + $(let mut $name = None;)* + + const EXPECTED_KEYS: &[&str] = &[$(stringify!($key),)*]; + const REQUIRED_KEYS: &[&str] = &[$(stringify!($req_key),)*]; + + let span = $input.span(); + let mut seen_keys = Vec::new(); + + while !$input.is_empty() { + let key = $input.call(Ident::parse_any)?; + + if seen_keys.contains(&key) { + Err(Error::new_spanned( + &key, + format!(r#"duplicated key "{key}". Keys can only be specified once."#), + ))? + } + + $input.parse::()?; + + match &*key.to_string() { + $( + stringify!($key) => $name = Some($parser), + )* + _ => { + Err(Error::new_spanned( + &key, + format!(r#"unknown key "{key}". Valid keys are: {EXPECTED_KEYS:?}."#), + ))? + } + } + + $input.parse::()?; + seen_keys.push(key); + } + + for key in REQUIRED_KEYS { + if !seen_keys.iter().any(|e| e == key) { + Err(Error::new(span, format!(r#"missing required key "{key}""#)))? + } + } + + let mut ordered_keys: Vec<&str> = Vec::new(); + for key in EXPECTED_KEYS { + if seen_keys.iter().any(|e| e == key) { + ordered_keys.push(key); + } + } + + if seen_keys != ordered_keys { + Err(Error::new( + span, + format!(r#"keys are not ordered as expected. Order them like: {ordered_keys:?}."#), + ))? + } + + $(let $req_name = $req_name.expect("required field");)* + }; + + // Handle required fields. + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $key:ident as $name:ident [required] => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)* [$name; $key]] $($rest)* + ) + }; + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $name:ident [required] => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)* [$name; $name]] $($rest)* + ) + }; + + // Handle optional fields. + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $key:ident as $name:ident => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)*] $($rest)* + ) + }; + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $name:ident => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)*] $($rest)* + ) + }; + + (from $input:expr; $($tok:tt)*) => { + parse_ordered_fields!(@gen [$input] [] [] $($tok)*) + } +} + +struct Parameter { + name: Ident, + ptype: Ident, + default: Expr, + description: LitStr, +} + +impl Parse for Parameter { + fn parse(input: ParseStream<'_>) -> Result { + let name = input.parse()?; + input.parse::()?; + let ptype = input.parse()?; + + let fields; + braced!(fields in input); + + parse_ordered_fields! { + from fields; + default [required] => fields.parse()?, + description [required] => fields.parse()?, + } + + Ok(Self { + name, + ptype, + default, + description, + }) + } +} + +pub(crate) struct ModuleInfo { + type_: Type, + license: AsciiLitStr, + name: AsciiLitStr, + authors: Option>, + description: Option, + alias: Option>, + firmware: Option>, + imports_ns: Option>, + params: Option>, +} + +impl Parse for ModuleInfo { + fn parse(input: ParseStream<'_>) -> Result { + parse_ordered_fields!( + from input; + type as type_ [required] => input.parse()?, + name [required] => input.parse()?, + authors => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + description => input.parse()?, + license [required] => input.parse()?, + alias => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + firmware => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + imports_ns => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + params => { + let list; + braced!(list in input); + Punctuated::parse_terminated(&list)? + }, + ); + + Ok(ModuleInfo { + type_, + license, + name, + authors, + description, + alias, + firmware, + imports_ns, + params, + }) + } +} + +pub(crate) fn module(info: ModuleInfo) -> Result { + let ModuleInfo { + type_, + license, + name, + authors, + description, + alias, + firmware, + imports_ns, + params: _, + } = &info; + + // Rust does not allow hyphens in identifiers, use underscore instead. + let ident = name.value().replace('-', "_"); + let mut modinfo = ModInfoBuilder::new(ident.as_ref()); + if let Some(authors) = authors { + for author in authors { + modinfo.emit("author", &author.value()); + } + } + if let Some(description) = description { + modinfo.emit("description", &description.value()); + } + modinfo.emit("license", &license.value()); + if let Some(aliases) = alias { + for alias in aliases { + modinfo.emit("alias", &alias.value()); + } + } + if let Some(firmware) = firmware { + for fw in firmware { + modinfo.emit("firmware", &fw.value()); + } + } + if let Some(imports) = imports_ns { + for ns in imports { + modinfo.emit("import_ns", &ns.value()); + } + } + + // Built-in modules also export the `file` modinfo string. + let file = + std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); + modinfo.emit_only_builtin("file", &file, false); + + modinfo.emit_params(&info); + + let modinfo_ts = modinfo.ts; + let params_ts = modinfo.param_ts; + + let ident_init = format_ident!("__{ident}_init"); + let ident_exit = format_ident!("__{ident}_exit"); + let ident_initcall = format_ident!("__{ident}_initcall"); + let initcall_section = ".initcall6.init"; + + let global_asm = format!( + r#".section "{initcall_section}", "a" + __{ident}_initcall: + .long __{ident}_init - . + .previous + "# + ); + + let name_cstr = CString::new(name.value()).expect("name contains NUL-terminator"); + + Ok(quote! { + /// The module name. + /// + /// Used by the printing macros, e.g. [`info!`]. + const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul(); + + /// The `LocalModule` type is the type of the module created by `module!`, + /// `module_pci_driver!`, `module_platform_driver!`, etc. + type LocalModule = #type_; + + impl ::kernel::ModuleMetadata for #type_ { + const NAME: &'static ::kernel::str::CStr = #name_cstr; + + #[cfg(MODULE)] + const THIS_MODULE: ::kernel::ThisModule = { + extern "C" { + static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; + } + + // SAFETY: `__this_module` is constructed by the kernel at load time + // and lives until the module is unloaded. + unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) } + }; + + #[cfg(not(MODULE))] + const THIS_MODULE: ::kernel::ThisModule = unsafe { + ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) + }; + } + + // Double nested modules, since then nobody can access the public items inside. + #[doc(hidden)] + mod __module_init { + mod __module_init { + use pin_init::PinInit; + + /// The "Rust loadable module" mark. + // + // This may be best done another way later on, e.g. as a new modinfo + // key or a new section. For the moment, keep it simple. + #[cfg(MODULE)] + #[used(compiler)] + static __IS_RUST_MODULE: () = (); + + static mut __MOD: ::core::mem::MaybeUninit = + ::core::mem::MaybeUninit::uninit(); + + // Loadable modules need to export the `{init,cleanup}_module` identifiers. + /// # Safety + /// + /// This function must not be called after module initialization, because it may be + /// freed after that completes. + #[cfg(MODULE)] + #[no_mangle] + #[link_section = ".init.text"] + pub unsafe extern "C" fn init_module() -> ::kernel::ffi::c_int { + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name. + unsafe { __init() } + } + + #[cfg(MODULE)] + #[used(compiler)] + #[link_section = ".init.data"] + static __UNIQUE_ID___addressable_init_module: unsafe extern "C" fn() -> i32 = + init_module; + + #[cfg(MODULE)] + #[no_mangle] + #[link_section = ".exit.text"] + pub extern "C" fn cleanup_module() { + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `init_module` has returned `0` + // (which delegates to `__init`). + unsafe { __exit() } + } + + #[cfg(MODULE)] + #[used(compiler)] + #[link_section = ".exit.data"] + static __UNIQUE_ID___addressable_cleanup_module: extern "C" fn() = cleanup_module; + + // Built-in modules are initialized through an initcall pointer + // and the identifiers need to be unique. + #[cfg(not(MODULE))] + #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] + #[link_section = #initcall_section] + #[used(compiler)] + pub static #ident_initcall: extern "C" fn() -> + ::kernel::ffi::c_int = #ident_init; + + #[cfg(not(MODULE))] + #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] + ::core::arch::global_asm!(#global_asm); + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern "C" fn #ident_init() -> ::kernel::ffi::c_int { + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // placement above in the initcall section. + unsafe { __init() } + } + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern "C" fn #ident_exit() { + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `#ident_init` has + // returned `0` (which delegates to `__init`). + unsafe { __exit() } + } + + /// # Safety + /// + /// This function must only be called once. + unsafe fn __init() -> ::kernel::ffi::c_int { + let initer = ::init( + ::kernel::module::this_module::() + ); + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__exit` cannot be called before or during `__init`. + match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } { + Ok(m) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// This function must + /// - only be called once, + /// - be called after `__init` has been called and returned `0`. + unsafe fn __exit() { + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__init` was already called. + unsafe { + // Invokes `drop()` on `__MOD`, which should be used for cleanup. + __MOD.assume_init_drop(); + } + } + + #modinfo_ts + } + } + + mod module_parameters { + #params_ts + } + }) +} diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs new file mode 100644 index 000000000..2181e312a --- /dev/null +++ b/rust/macros/paste.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; + +fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> { + let mut tokens = tokens.iter(); + let mut segments = Vec::new(); + let mut span = None; + loop { + match tokens.next() { + None => break, + Some(TokenTree::Literal(lit)) => { + // Allow us to concat string literals by stripping quotes + let mut value = lit.to_string(); + if value.starts_with('"') && value.ends_with('"') { + value.remove(0); + value.pop(); + } + segments.push((value, lit.span())); + } + Some(TokenTree::Ident(ident)) => { + let mut value = ident.to_string(); + if value.starts_with("r#") { + value.replace_range(0..2, ""); + } + segments.push((value, ident.span())); + } + Some(TokenTree::Punct(p)) if p.as_char() == ':' => { + let Some(TokenTree::Ident(ident)) = tokens.next() else { + panic!("expected identifier as modifier"); + }; + + let (mut value, sp) = segments.pop().expect("expected identifier before modifier"); + match ident.to_string().as_str() { + // Set the overall span of concatenated token as current span + "span" => { + assert!( + span.is_none(), + "span modifier should only appear at most once" + ); + span = Some(sp); + } + "lower" => value = value.to_lowercase(), + "upper" => value = value.to_uppercase(), + v => panic!("unknown modifier `{v}`"), + }; + segments.push((value, sp)); + } + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => { + let tokens = group.stream().into_iter().collect::>(); + segments.append(&mut concat_helper(tokens.as_slice())); + } + token => panic!("unexpected token in paste segments: {token:?}"), + }; + } + + segments +} + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let segments = concat_helper(tokens); + let pasted: String = segments.into_iter().map(|x| x.0).collect(); + TokenTree::Ident(Ident::new(&pasted, group_span)) +} + +pub(crate) fn expand(tokens: &mut Vec) { + for token in tokens.iter_mut() { + if let TokenTree::Group(group) = token { + let delimiter = group.delimiter(); + let span = group.span(); + let mut stream: Vec<_> = group.stream().into_iter().collect(); + // Find groups that looks like `[< A B C D >]` + if delimiter == Delimiter::Bracket + && stream.len() >= 3 + && matches!(&stream[0], TokenTree::Punct(p) if p.as_char() == '<') + && matches!(&stream[stream.len() - 1], TokenTree::Punct(p) if p.as_char() == '>') + { + // Replace the group with concatenated token + *token = concat(&stream[1..stream.len() - 1], span); + } else { + // Recursively expand tokens inside the group + expand(&mut stream); + let mut group = Group::new(delimiter, stream.into_iter().collect()); + group.set_span(span); + *token = TokenTree::Group(group); + } + } + } + + // Path segments cannot contain invisible delimiter group, so remove them if any. + for i in (0..tokens.len().saturating_sub(3)).rev() { + // Looking for a double colon + if matches!( + (&tokens[i + 1], &tokens[i + 2]), + (TokenTree::Punct(a), TokenTree::Punct(b)) + if a.as_char() == ':' && a.spacing() == Spacing::Joint && b.as_char() == ':' + ) { + match &tokens[i + 3] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i + 3..i + 4, group.stream()); + } + _ => (), + } + + match &tokens[i] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i..i + 1, group.stream()); + } + _ => (), + } + } + } +} diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs new file mode 100644 index 000000000..be9a5ed8a --- /dev/null +++ b/rust/macros/vtable.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::{ + collections::HashSet, + iter::Extend, // +}; + +use proc_macro2::{ + Ident, + TokenStream, // +}; +use quote::ToTokens; +use syn::{ + parse_quote, + Error, + ImplItem, + Item, + ItemImpl, + ItemTrait, + Result, + TraitItem, // +}; + +fn handle_trait(mut item: ItemTrait) -> Result { + let mut gen_items = Vec::new(); + + gen_items.push(parse_quote! { + /// A marker to prevent implementors from forgetting to use [`#[vtable]`](vtable) + /// attribute when implementing this trait. + const USE_VTABLE_ATTR: (); + }); + + // Add `type OwnerModule: ModuleMetadata` as a required associated type if + // the trait does not already define it. + if !item + .items + .iter() + .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule")) + { + gen_items.push(parse_quote! { + /// The module implementing this vtable trait. + /// + /// Automatically set to `crate::LocalModule` by the `#[vtable]` + /// impl macro. + type OwnerModule: ::kernel::ModuleMetadata; + }); + } + + for item in &item.items { + if let TraitItem::Fn(fn_item) = item { + let name = &fn_item.sig.ident; + let gen_const_name = Ident::new( + &format!("HAS_{}", name.to_string().to_uppercase()), + name.span(), + ); + + // We don't know on the implementation-site whether a method is required or provided + // so we have to generate a const for all methods. + let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); + let comment = + format!("Indicates if the `{name}` method is overridden by the implementor."); + gen_items.push(parse_quote! { + #(#cfg_attrs)* + #[doc = #comment] + const #gen_const_name: bool = false; + }); + } + } + + item.items.extend(gen_items); + Ok(item) +} + +fn handle_impl(mut item: ItemImpl) -> Result { + let mut gen_items = Vec::new(); + let mut defined_items = HashSet::new(); + + // Iterate over all user-defined items to gather any possible explicit overrides. + for item in &item.items { + match item { + ImplItem::Const(const_item) => { + defined_items.insert(const_item.ident.clone()); + } + ImplItem::Type(type_item) => { + defined_items.insert(type_item.ident.clone()); + } + _ => {} + } + } + + gen_items.push(parse_quote! { + const USE_VTABLE_ATTR: () = (); + }); + + // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined. + // `crate::LocalModule` resolves to the real module type (via `module!`) or a + // dummy fallback in non-module contexts (e.g., doctests). + if !defined_items.contains(&parse_quote!(OwnerModule)) { + gen_items.push(parse_quote! { + type OwnerModule = crate::LocalModule; + }); + } + + for item in &item.items { + if let ImplItem::Fn(fn_item) = item { + let name = &fn_item.sig.ident; + let gen_const_name = Ident::new( + &format!("HAS_{}", name.to_string().to_uppercase()), + name.span(), + ); + // Skip if it's declared already -- this allows user override. + if defined_items.contains(&gen_const_name) { + continue; + } + let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); + gen_items.push(parse_quote! { + #(#cfg_attrs)* + const #gen_const_name: bool = true; + }); + } + } + + item.items.extend(gen_items); + Ok(item) +} + +pub(crate) fn vtable(input: Item) -> Result { + match input { + Item::Trait(item) => Ok(handle_trait(item)?.into_token_stream()), + Item::Impl(item) => Ok(handle_impl(item)?.into_token_stream()), + _ => Err(Error::new_spanned( + input, + "`#[vtable]` attribute should only be applied to trait or impl block", + ))?, + } +} diff --git a/rust/pin-init/CONTRIBUTING.md b/rust/pin-init/CONTRIBUTING.md new file mode 100644 index 000000000..16c899a7a --- /dev/null +++ b/rust/pin-init/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contributing to `pin-init` + +Thanks for showing interest in contributing to `pin-init`! This document outlines the guidelines for +contributing to `pin-init`. + +All contributions are double-licensed under Apache 2.0 and MIT. You can find the respective licenses +in the `LICENSE-APACHE` and `LICENSE-MIT` files. + +## Non-Code Contributions + +### Bug Reports + +For any type of bug report, please submit an issue using the bug report issue template. + +If the issue is a soundness issue, please privately report it as a security vulnerability via the +GitHub web interface. + +### Feature Requests + +If you have any feature requests, please submit an issue using the feature request issue template. + +### Questions and Getting Help + +You can ask questions in the Discussions page of the GitHub repository. If you're encountering +problems or just have questions related to `pin-init` in the Linux kernel, you can also ask your +questions in the [Rust-for-Linux Zulip](https://rust-for-linux.zulipchat.com/) or see +. + +## Contributing Code + +### Linux Kernel + +`pin-init` is used by the Linux kernel and all commits are synchronized to it. For this reason, the +same requirements for commits apply to `pin-init`. See [the kernel's documentation] for details. The +rest of this document will also cover some of the rules listed there and additional ones. + +[the kernel's documentation]: https://docs.kernel.org/process/submitting-patches.html + +Contributions to `pin-init` ideally go through the [GitHub repository], because that repository runs +a CI with lots of tests not present in the kernel. However, patches are also accepted (though not +preferred). Do note that there are some files that are only present in the GitHub repository such as +tests, licenses and cargo related files. Making changes to them can only happen via GitHub. + +[GitHub repository]: https://github.com/Rust-for-Linux/pin-init + +### Commit Style + +Everything must compile without errors or warnings and all tests must pass after **every commit**. +This is important for bisection and also required by the kernel. + +Each commit should be a single, logically cohesive change. Of course it's best to keep the changes +small and digestible, but logically linked changes should be made in the same commit. For example, +when fixing typos, create a single commit that fixes all of them instead of one commit per typo. + +Commits must have a meaningful commit title. Commits with changes to files in the `internal` +directory should have a title prefixed with `internal:`. The commit message should explain the +change and its rationale. You also have to add your `Signed-off-by` tag, see [Developer's +Certificate of Origin]. This has to be done for both mailing list submissions as well as GitHub +submissions. + +[Developer's Certificate of Origin]: https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin + +Any changes made to public APIs must be documented not only in the commit message, but also in the +`CHANGELOG.md` file. This is especially important for breaking changes, as those warrant a major +version bump. + +If you make changes to the top-level crate documentation, you also need to update the `README.md` +via `cargo rdme`. + +Some of these rules can be ignored if the change is done solely to files that are not present in the +kernel version of this library. Those files are documented in the `sync-kernel.sh` script at the +very bottom in the `--exclude` flag given to the `git am` command. diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md new file mode 100644 index 000000000..2312c9e75 --- /dev/null +++ b/rust/pin-init/README.md @@ -0,0 +1,247 @@ +[![Crates.io](https://img.shields.io/crates/v/pin-init.svg)](https://crates.io/crates/pin-init) +[![Documentation](https://docs.rs/pin-init/badge.svg)](https://docs.rs/pin-init/) +[![Dependency status](https://deps.rs/repo/github/Rust-for-Linux/pin-init/status.svg)](https://deps.rs/repo/github/Rust-for-Linux/pin-init) +![License](https://img.shields.io/crates/l/pin-init) +[![Toolchain](https://img.shields.io/badge/toolchain-nightly-red)](#nightly-only) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Rust-for-Linux/pin-init/ci.yml) +# `pin-init` + +> [!NOTE] +> +> This crate was originally named [`pinned-init`], but the migration to +> `pin-init` is not yet complete. The `legacy` branch contains the current +> version of the `pinned-init` crate & the `main` branch already incorporates +> the rename to `pin-init`. +> +> There are still some changes needed on the kernel side before the migration +> can be completed. + +[`pinned-init`]: https://crates.io/crates/pinned-init + + + +Library to safely and fallibly initialize pinned `struct`s using in-place constructors. + +[Pinning][pinning] is Rust's way of ensuring data does not move. + +It also allows in-place initialization of big `struct`s that would otherwise produce a stack +overflow. + +This library's main use-case is in [Rust-for-Linux]. Although this version can be used +standalone. + +There are cases when you want to in-place initialize a struct. For example when it is very big +and moving it from the stack is not an option, because it is bigger than the stack itself. +Another reason would be that you need the address of the object to initialize it. This stands +in direct conflict with Rust's normal process of first initializing an object and then moving +it into it's final memory location. For more information, see +. + +This library allows you to do in-place initialization safely. + +### Nightly Needed for `alloc` feature + +This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +enabled and thus this feature can only be used with a nightly compiler. When enabling the +`alloc` feature, the user will be required to activate `allocator_api` as well. + +[`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html + +The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +mode. + +### Nightly needed for `unsafe-pinned` feature + +This feature enables the `Wrapper` implementation on the unstable `core::pin::UnsafePinned` type. +This requires the [`unsafe_pinned` unstable feature](https://github.com/rust-lang/rust/issues/125735) +and therefore a nightly compiler. Note that this feature is not enabled by default. + +## Overview + +To initialize a `struct` with an in-place constructor you will need two things: +- an in-place constructor, +- a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], + [`Box`] or any other smart pointer that supports this library). + +To get an in-place constructor there are generally three options: +- directly creating an in-place constructor using the [`pin_init!`] macro, +- a custom function/macro returning an in-place constructor provided by someone else, +- using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. + +Aside from pinned initialization, this library also supports in-place construction without +pinning, the macros/types/functions are generally named like the pinned variants without the +`pin_` prefix. + +## Examples + +Throughout the examples we will often make use of the `CMutex` type which can be found in +`../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +requires it to be pinned to be locked and thus is a prime candidate for using this library. + +### Using the [`pin_init!`] macro + +If you want to use [`PinInit`], then you will have to annotate your `struct` with +`#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +[structurally pinned fields]. After doing this, you can then create an in-place constructor via +[`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +that you need to write `<-` instead of `:` for fields that you want to initialize in-place. + +```rust +use pin_init::{pin_data, pin_init, InPlaceInit}; + +#[pin_data] +struct Foo { + #[pin] + a: CMutex, + b: u32, +} + +let foo = pin_init!(Foo { + a <- CMutex::new(42), + b: 24, +}); +``` + +`foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like +(or just the stack) to actually initialize a `Foo`: + +```rust +let foo: Result>, AllocError> = Box::pin_init(foo); +``` + +For more information see the [`pin_init!`] macro. + +### Using a custom function/macro that returns an initializer + +Many types that use this library supply a function/macro that returns an initializer, because +the above method only works for types where you can access the fields. + +```rust +let mtx: Result>>, _> = Arc::pin_init(CMutex::new(42)); +``` + +To declare an init macro/function you just return an [`impl PinInit`]: + +```rust +#[pin_data] +struct DriverData { + #[pin] + status: CMutex, + buffer: Box<[u8; 1_000_000]>, +} + +impl DriverData { + fn new() -> impl PinInit { + pin_init!(Self { + status <- CMutex::new(0), + buffer: Box::init(pin_init::init_zeroed())?, + }? Error) + } +} +``` + +### Manual creation of an initializer + +Often when working with primitives the previous approaches are not sufficient. That is where +[`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +[`impl PinInit`] directly from a closure. Of course you have to ensure that the closure +actually does the initialization in the correct way. Here are the things to look out for +(we are calling the parameter to the closure `slot`): +- when the closure returns `Ok(())`, then it has completed the initialization successfully, so + `slot` now contains a valid bit pattern for the type `T`, +- when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so + you need to take care to clean up anything if your initialization fails mid-way, +- you may assume that `slot` will stay pinned even after the closure returns until `drop` of + `slot` gets called. + +```rust +use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +use core::{ + marker::PhantomPinned, + cell::UnsafeCell, + pin::Pin, + mem::MaybeUninit, +}; +mod bindings { + #[repr(C)] + pub struct foo { + /* fields from C ... */ + } + extern "C" { + pub fn init_foo(ptr: *mut foo); + pub fn destroy_foo(ptr: *mut foo); + #[must_use = "you must check the error return code"] + pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; + } +} + +/// # Invariants +/// +/// `foo` is always initialized +#[pin_data(PinnedDrop)] +pub struct RawFoo { + #[pin] + _p: PhantomPinned, + #[pin] + foo: UnsafeCell>, +} + +impl RawFoo { + pub fn new(flags: u32) -> impl PinInit { + // SAFETY: + // - when the closure returns `Ok(())`, then it has successfully initialized and + // enabled `foo`, + // - when it returns `Err(e)`, then it has cleaned up before + unsafe { + pin_init_from_closure(move |slot: *mut Self| { + // `slot` contains uninit memory, avoid creating a reference. + let foo = &raw mut (*slot).foo; + let foo = UnsafeCell::raw_get(foo).cast::(); + + // Initialize the `foo` + bindings::init_foo(foo); + + // Try to enable it. + let err = bindings::enable_foo(foo, flags); + if err != 0 { + // Enabling has failed, first clean up the foo and then return the error. + bindings::destroy_foo(foo); + Err(err) + } else { + // All fields of `RawFoo` have been initialized, since `_p` is a ZST. + Ok(()) + } + }) + } + } +} + +#[pinned_drop] +impl PinnedDrop for RawFoo { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Since `foo` is initialized, destroying is safe. + unsafe { bindings::destroy_foo(self.foo.get().cast::()) }; + } +} +``` + +For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +the `kernel` crate. The [`sync`] module is a good starting point. + +[`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +[pinning]: https://doc.rust-lang.org/std/pin/index.html +[structurally pinned fields]: https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning +[stack]: https://docs.rs/pin-init/latest/pin_init/macro.stack_pin_init.html +[`impl PinInit`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl PinInit`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl Init`]: https://docs.rs/pin-init/latest/pin_init/trait.Init.html +[Rust-for-Linux]: https://rust-for-linux.com/ + + + + +[`Arc`]: https://doc.rust-lang.org/stable/alloc/sync/struct.Arc.html +[`Box`]: https://doc.rust-lang.org/stable/alloc/boxed/struct.Box.html diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs new file mode 100644 index 000000000..c05139927 --- /dev/null +++ b/rust/pin-init/examples/big_struct_in_place.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use pin_init::*; + +// Struct with size over 1GiB +#[derive(Debug)] +#[allow(dead_code)] +pub struct BigStruct { + buf: [u8; 1024 * 1024 * 1024], + a: u64, + b: u64, + c: u64, + d: u64, + managed_buf: ManagedBuf, +} + +#[derive(Debug)] +pub struct ManagedBuf { + buf: [u8; 1024 * 1024], +} + +impl ManagedBuf { + pub fn new() -> impl Init { + init!(ManagedBuf { buf <- init_zeroed() }) + } +} + +fn main() { + #[cfg(any(feature = "std", feature = "alloc"))] + { + // we want to initialize the struct in-place, otherwise we would get a stackoverflow + let buf: Box = Box::init(init!(BigStruct { + buf <- init_zeroed(), + a: 7, + b: 186, + c: 7789, + d: 34, + managed_buf <- ManagedBuf::new(), + })) + .unwrap(); + println!("{}", core::mem::size_of_val(&*buf)); + } +} diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs new file mode 100644 index 000000000..96f095398 --- /dev/null +++ b/rust/pin-init/examples/error.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::convert::Infallible; + +#[cfg(feature = "alloc")] +use std::alloc::AllocError; + +#[derive(Debug)] +pub struct Error; + +impl From for Error { + #[inline] + fn from(e: Infallible) -> Self { + match e {} + } +} + +#[cfg(feature = "alloc")] +impl From for Error { + #[inline] + fn from(_: AllocError) -> Self { + Self + } +} + +#[allow(dead_code)] +fn main() { + let _ = Error; +} diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs new file mode 100644 index 000000000..424585fe2 --- /dev/null +++ b/rust/pin-init/examples/linked_list.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::Cell, + marker::PhantomPinned, + pin::Pin, + ptr::{self, NonNull}, +}; + +use pin_init::*; + +#[allow(unused_attributes)] +mod error; +#[allow(unused_imports)] +use error::Error; + +#[pin_data(PinnedDrop)] +#[repr(C)] +#[derive(Debug)] +pub struct ListHead { + next: Link, + prev: Link, + #[pin] + pin: PhantomPinned, +} + +impl ListHead { + #[inline] + pub fn new() -> impl PinInit { + pin_init!(&this in Self { + next: unsafe { Link::new_unchecked(this) }, + prev: unsafe { Link::new_unchecked(this) }, + pin: PhantomPinned, + }) + } + + #[inline] + #[allow(dead_code)] + pub fn insert_next(list: &ListHead) -> impl PinInit + '_ { + pin_init!(&this in Self { + prev: list.next.prev().replace(unsafe { Link::new_unchecked(this)}), + next: list.next.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }) + } + + #[inline] + pub fn insert_prev(list: &ListHead) -> impl PinInit + '_ { + pin_init!(&this in Self { + next: list.prev.next().replace(unsafe { Link::new_unchecked(this)}), + prev: list.prev.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }) + } + + #[inline] + pub fn next(&self) -> Option> { + if ptr::eq(self.next.as_ptr(), self) { + None + } else { + Some(unsafe { NonNull::new_unchecked(self.next.as_ptr() as *mut Self) }) + } + } + + #[allow(dead_code)] + pub fn size(&self) -> usize { + let mut size = 1; + let mut cur = self.next.clone(); + while !ptr::eq(self, cur.cur()) { + cur = cur.next().clone(); + size += 1; + } + size + } +} + +#[pinned_drop] +impl PinnedDrop for ListHead { + //#[inline] + fn drop(self: Pin<&mut Self>) { + if !ptr::eq(self.next.as_ptr(), &*self) { + let next = unsafe { &*self.next.as_ptr() }; + let prev = unsafe { &*self.prev.as_ptr() }; + next.prev.set(&self.prev); + prev.next.set(&self.next); + } + } +} + +#[repr(transparent)] +#[derive(Clone, Debug)] +struct Link(Cell>); + +impl Link { + /// # Safety + /// + /// The contents of the pointer should form a consistent circular + /// linked list; for example, a "next" link should be pointed back + /// by the target `ListHead`'s "prev" link and a "prev" link should be + /// pointed back by the target `ListHead`'s "next" link. + #[inline] + unsafe fn new_unchecked(ptr: NonNull) -> Self { + Self(Cell::new(ptr)) + } + + #[inline] + fn next(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).next } + } + + #[inline] + #[allow(dead_code)] + fn prev(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).prev } + } + + #[allow(dead_code)] + fn cur(&self) -> &ListHead { + unsafe { &*self.0.get().as_ptr() } + } + + #[inline] + fn replace(&self, other: Link) -> Link { + unsafe { Link::new_unchecked(self.0.replace(other.0.get())) } + } + + #[inline] + fn as_ptr(&self) -> *const ListHead { + self.0.get().as_ptr() + } + + #[inline] + fn set(&self, val: &Link) { + self.0.set(val.0.get()); + } +} + +#[allow(dead_code)] +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() -> Result<(), Error> { + let a = Box::pin_init(ListHead::new())?; + stack_pin_init!(let b = ListHead::insert_next(&a)); + stack_pin_init!(let c = ListHead::insert_next(&a)); + stack_pin_init!(let d = ListHead::insert_next(&b)); + let e = Box::pin_init(ListHead::insert_next(&b))?; + println!("a ({a:p}): {a:?}"); + println!("b ({b:p}): {b:?}"); + println!("c ({c:p}): {c:?}"); + println!("d ({d:p}): {d:?}"); + println!("e ({e:p}): {e:?}"); + let mut inspect = &*a; + while let Some(next) = inspect.next() { + println!("({inspect:p}): {inspect:?}"); + inspect = unsafe { &*next.as_ptr() }; + if core::ptr::eq(inspect, &*a) { + break; + } + } + Ok(()) +} diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs new file mode 100644 index 000000000..e8d4dbb66 --- /dev/null +++ b/rust/pin-init/examples/mutex.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(clippy::missing_safety_doc)] + +use core::{ + cell::{Cell, UnsafeCell}, + marker::PhantomPinned, + ops::{Deref, DerefMut}, + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, +}; +#[cfg(feature = "std")] +use std::{ + sync::Arc, + thread::{self, sleep, Builder, Thread}, + time::Duration, +}; + +use pin_init::*; +#[allow(unused_attributes)] +#[path = "./linked_list.rs"] +pub mod linked_list; +use linked_list::*; + +pub struct SpinLock { + inner: AtomicBool, +} + +impl SpinLock { + #[inline] + pub fn acquire(&self) -> SpinLockGuard<'_> { + while self + .inner + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + #[cfg(feature = "std")] + while self.inner.load(Ordering::Relaxed) { + thread::yield_now(); + } + } + SpinLockGuard(self) + } + + #[inline] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + inner: AtomicBool::new(false), + } + } +} + +pub struct SpinLockGuard<'a>(&'a SpinLock); + +impl Drop for SpinLockGuard<'_> { + #[inline] + fn drop(&mut self) { + self.0.inner.store(false, Ordering::Release); + } +} + +#[pin_data] +pub struct CMutex { + #[pin] + wait_list: ListHead, + spin_lock: SpinLock, + locked: Cell, + #[pin] + data: UnsafeCell, +} + +impl CMutex { + #[inline] + pub fn new(val: impl PinInit) -> impl PinInit { + pin_init!(CMutex { + wait_list <- ListHead::new(), + spin_lock: SpinLock::new(), + locked: Cell::new(false), + data <- UnsafeCell::pin_init(val), + }) + } + + #[inline] + pub fn lock(&self) -> Pin> { + let mut sguard = self.spin_lock.acquire(); + if self.locked.get() { + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); + // println!("wait list length: {}", self.wait_list.size()); + while self.locked.get() { + drop(sguard); + #[cfg(feature = "std")] + thread::park(); + sguard = self.spin_lock.acquire(); + } + } + self.locked.set(true); + unsafe { + Pin::new_unchecked(CMutexGuard { + mtx: self, + _pin: PhantomPinned, + }) + } + } + + #[allow(dead_code)] + pub fn get_data_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: we have an exclusive reference and thus nobody has access to data. + unsafe { &mut *self.data.get() } + } +} + +unsafe impl Send for CMutex {} +unsafe impl Sync for CMutex {} + +pub struct CMutexGuard<'a, T> { + mtx: &'a CMutex, + _pin: PhantomPinned, +} + +impl Drop for CMutexGuard<'_, T> { + #[inline] + fn drop(&mut self) { + let sguard = self.mtx.spin_lock.acquire(); + self.mtx.locked.set(false); + if let Some(list_field) = self.mtx.wait_list.next() { + let _wait_entry = list_field.as_ptr().cast::(); + #[cfg(feature = "std")] + unsafe { + (*_wait_entry).thread.unpark() + }; + } + drop(sguard); + } +} + +impl Deref for CMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } +} + +impl DerefMut for CMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } +} + +#[pin_data] +#[repr(C)] +struct WaitEntry { + #[pin] + wait_list: ListHead, + #[cfg(feature = "std")] + thread: Thread, +} + +impl WaitEntry { + #[inline] + fn insert_new(list: &ListHead) -> impl PinInit + '_ { + #[cfg(feature = "std")] + { + pin_init!(Self { + thread: thread::current(), + wait_list <- ListHead::insert_prev(list), + }) + } + #[cfg(not(feature = "std"))] + { + pin_init!(Self { + wait_list <- ListHead::insert_prev(list), + }) + } + } +} + +#[cfg_attr(test, test)] +#[allow(dead_code)] +fn main() { + #[cfg(feature = "std")] + { + let mtx: Pin>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = if cfg!(miri) { 100 } else { 1_000 }; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", *mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs new file mode 100644 index 000000000..00f457e68 --- /dev/null +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// inspired by +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +#[cfg(not(windows))] +mod pthread_mtx { + #[cfg(feature = "alloc")] + use core::alloc::AllocError; + use core::{ + cell::UnsafeCell, + marker::PhantomPinned, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + pin::Pin, + }; + use pin_init::*; + use std::convert::Infallible; + + #[pin_data(PinnedDrop)] + pub struct PThreadMutex { + #[pin] + raw: UnsafeCell, + data: UnsafeCell, + #[pin] + pin: PhantomPinned, + } + + unsafe impl Send for PThreadMutex {} + unsafe impl Sync for PThreadMutex {} + + #[pinned_drop] + impl PinnedDrop for PThreadMutex { + fn drop(self: Pin<&mut Self>) { + unsafe { + libc::pthread_mutex_destroy(self.raw.get()); + } + } + } + + #[derive(Debug)] + pub enum Error { + #[allow(dead_code)] + IO(std::io::Error), + #[allow(dead_code)] + Alloc, + } + + impl From for Error { + fn from(e: Infallible) -> Self { + match e {} + } + } + + #[cfg(feature = "alloc")] + impl From for Error { + fn from(_: AllocError) -> Self { + Self::Alloc + } + } + + impl PThreadMutex { + #[allow(dead_code)] + pub fn new(data: T) -> impl PinInit { + fn init_raw() -> impl PinInit, Error> { + let init = |slot: *mut UnsafeCell| { + // we can cast, because `UnsafeCell` has the same layout as T. + let slot: *mut libc::pthread_mutex_t = slot.cast(); + let mut attr = MaybeUninit::uninit(); + let attr = attr.as_mut_ptr(); + // SAFETY: ptr is valid + let ret = unsafe { libc::pthread_mutexattr_init(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: attr is initialized + let ret = unsafe { + libc::pthread_mutexattr_settype(attr, libc::PTHREAD_MUTEX_NORMAL) + }; + if ret != 0 { + // SAFETY: attr is initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: slot is valid + unsafe { slot.write(libc::PTHREAD_MUTEX_INITIALIZER) }; + // SAFETY: attr and slot are valid ptrs and attr is initialized + let ret = unsafe { libc::pthread_mutex_init(slot, attr) }; + // SAFETY: attr was initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + Ok(()) + }; + // SAFETY: mutex has been initialized + unsafe { pin_init_from_closure(init) } + } + pin_init!(Self { + data: UnsafeCell::new(data), + raw <- init_raw(), + pin: PhantomPinned, + }? Error) + } + + #[allow(dead_code)] + pub fn lock(&self) -> PThreadMutexGuard<'_, T> { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_lock(self.raw.get()) }; + PThreadMutexGuard { mtx: self } + } + } + + pub struct PThreadMutexGuard<'a, T> { + mtx: &'a PThreadMutex, + } + + impl Drop for PThreadMutexGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_unlock(self.mtx.raw.get()) }; + } + } + + impl Deref for PThreadMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } + } + + impl DerefMut for PThreadMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } + } +} + +#[cfg_attr(test, test)] +#[cfg_attr(all(test, miri), ignore)] +fn main() { + #[cfg(all(any(feature = "std", feature = "alloc"), not(windows)))] + { + use core::pin::Pin; + use pin_init::*; + use pthread_mtx::*; + use std::{ + sync::Arc, + thread::{sleep, Builder}, + time::Duration, + }; + let mtx: Pin>> = Arc::try_pin_init(PThreadMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", *mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs new file mode 100644 index 000000000..8dd52313c --- /dev/null +++ b/rust/pin-init/examples/static_init.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(unused_imports)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit, + ops, + pin::Pin, + time::Duration, +}; +use pin_init::*; +#[cfg(feature = "std")] +use std::{ + sync::Arc, + thread::{sleep, Builder}, +}; + +#[allow(unused_attributes)] +mod mutex; +use mutex::*; + +pub struct StaticInit { + cell: UnsafeCell>, + init: Cell>, + lock: SpinLock, + present: Cell, +} + +unsafe impl Sync for StaticInit {} +unsafe impl Send for StaticInit {} + +impl> StaticInit { + pub const fn new(init: I) -> Self { + Self { + cell: UnsafeCell::new(MaybeUninit::uninit()), + init: Cell::new(Some(init)), + lock: SpinLock::new(), + present: Cell::new(false), + } + } +} + +impl> ops::Deref for StaticInit { + type Target = T; + fn deref(&self) -> &Self::Target { + if self.present.get() { + unsafe { (*self.cell.get()).assume_init_ref() } + } else { + println!("acquire spinlock on static init"); + let _guard = self.lock.acquire(); + println!("rechecking present..."); + std::thread::sleep(std::time::Duration::from_millis(200)); + if self.present.get() { + return unsafe { (*self.cell.get()).assume_init_ref() }; + } + println!("doing init"); + let ptr = self.cell.get().cast::(); + match self.init.take() { + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, + None => unsafe { core::hint::unreachable_unchecked() }, + } + self.present.set(true); + unsafe { (*self.cell.get()).assume_init_ref() } + } + } +} + +pub struct CountInit; + +unsafe impl PinInit> for CountInit { + unsafe fn __init(self, slot: *mut CMutex) -> Result<(), core::convert::Infallible> { + let init = CMutex::new(0); + std::thread::sleep(std::time::Duration::from_millis(1000)); + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) + } +} + +pub static COUNT: StaticInit, CountInit> = StaticInit::new(CountInit); + +fn main() { + #[cfg(feature = "std")] + { + let mtx: Pin>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *COUNT.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *COUNT.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}, {:?}", *mtx.lock(), *COUNT.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs new file mode 100644 index 000000000..c7d9b3e62 --- /dev/null +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::fmt::Display; + +use proc_macro2::TokenStream; +use quote::quote_spanned; +use syn::{spanned::Spanned, Error}; + +pub(crate) struct DiagCtxt(TokenStream); +pub(crate) struct ErrorGuaranteed(()); + +impl DiagCtxt { + pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed { + let error = Error::new(span.span(), msg); + self.0.extend(error.into_compile_error()); + ErrorGuaranteed(()) + } + + pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) { + // Have the message start on a new line for visual clarity. + let msg = format!("\n{}", msg); + self.0.extend(quote_spanned!(span.span() => + // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable. + const _: () = { + #[deprecated = #msg] + const fn warn() {} + warn(); + }; + )); + } + + pub(crate) fn with( + fun: impl FnOnce(&mut DiagCtxt) -> Result, + ) -> TokenStream { + let mut dcx = Self(TokenStream::new()); + match fun(&mut dcx) { + Ok(mut stream) => { + stream.extend(dcx.0); + stream + } + Err(ErrorGuaranteed(())) => dcx.0, + } + } +} diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs new file mode 100644 index 000000000..fd0b5ea4a --- /dev/null +++ b/rust/pin-init/internal/src/init.rs @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote}; +use syn::{ + braced, + parse::{End, Parse}, + parse_quote, + punctuated::Punctuated, + spanned::Spanned, + token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Path, Token, Type, +}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) struct Initializer { + attrs: Vec, + this: Option, + path: Path, + brace_token: token::Brace, + fields: Punctuated, + rest: Option<(Token![..], Expr)>, + error: Option<(Token![?], Type)>, +} + +struct This { + _and_token: Token![&], + ident: Ident, + _in_token: Token![in], +} + +struct InitializerField { + attrs: Vec, + kind: InitializerKind, +} + +enum InitializerKind { + Value { + ident: Ident, + value: Option<(Token![:], Expr)>, + }, + Init { + ident: Ident, + _left_arrow_token: Token![<-], + value: Expr, + }, + Code { + _underscore_token: Token![_], + _colon_token: Token![:], + block: Block, + }, +} + +impl InitializerKind { + fn ident(&self) -> Option<&Ident> { + match self { + Self::Value { ident, .. } | Self::Init { ident, .. } => Some(ident), + Self::Code { .. } => None, + } + } +} + +enum InitializerAttribute { + DefaultError(DefaultErrorAttribute), +} + +struct DefaultErrorAttribute { + ty: Box, +} + +pub(crate) fn expand( + Initializer { + attrs, + this, + path, + brace_token, + fields, + rest, + error, + }: Initializer, + default_error: Option<&'static str>, + pinned: bool, + dcx: &mut DiagCtxt, +) -> Result { + let error = error.map_or_else( + || { + if let Some(default_error) = attrs.iter().fold(None, |acc, attr| { + #[expect(irrefutable_let_patterns)] + if let InitializerAttribute::DefaultError(DefaultErrorAttribute { ty }) = attr { + Some(ty.clone()) + } else { + acc + } + }) { + default_error + } else if let Some(default_error) = default_error { + syn::parse_str(default_error).unwrap() + } else { + dcx.error(brace_token.span.close(), "expected `? ` after `}`"); + parse_quote!(::core::convert::Infallible) + } + }, + |(_, err)| Box::new(err), + ); + let slot = format_ident!("slot"); + let (has_data_trait, get_data, init_from_closure) = if pinned { + ( + format_ident!("HasPinData"), + format_ident!("__pin_data"), + format_ident!("pin_init_from_closure"), + ) + } else { + ( + format_ident!("HasInitData"), + format_ident!("__init_data"), + format_ident!("init_from_closure"), + ) + }; + let init_kind = get_init_kind(rest, dcx); + let zeroable_check = match init_kind { + InitKind::Normal => quote!(), + InitKind::Zeroing => quote! { + // The user specified `..Zeroable::zeroed()` at the end of the list of fields. + // Therefore we check if the struct implements `Zeroable` and then zero the memory. + // This allows us to also remove the check that all fields are present (since we + // already set the memory to zero and that is a valid bit pattern). + fn assert_zeroable(_: *mut T) + where T: ::pin_init::Zeroable + {} + // Ensure that the struct is indeed `Zeroable`. + assert_zeroable(#slot); + // SAFETY: The type implements `Zeroable` by the check above. + unsafe { ::core::ptr::write_bytes(#slot, 0, 1) }; + }, + }; + let this = match this { + None => quote!(), + Some(This { ident, .. }) => quote! { + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + let #ident = unsafe { ::core::ptr::NonNull::new_unchecked(slot) }; + }, + }; + // `mixed_site` ensures that the data is not accessible to the user-controlled code. + let data = Ident::new("__data", Span::mixed_site()); + let init_fields = init_fields(&fields, pinned, &data, &slot); + let field_check = make_field_check(&fields, init_kind, &path); + Ok(quote! {{ + // Get the data about fields from the supplied type. + // SAFETY: TODO + let #data = unsafe { + use ::pin_init::__internal::#has_data_trait; + // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit + // generics (which need to be present with that syntax). + #path::#get_data() + }; + // Ensure that `#data` really is of type `#data` and help with type inference: + let init = #data.__make_closure::<_, #error>( + move |slot| { + #zeroable_check + #this + #init_fields + #field_check + // SAFETY: we are the `init!` macro that is allowed to call this. + Ok(unsafe { ::pin_init::__internal::InitOk::new() }) + } + ); + let init = move |slot| -> ::core::result::Result<(), #error> { + init(slot).map(|__InitOk| ()) + }; + // SAFETY: TODO + unsafe { ::pin_init::#init_from_closure::<_, #error>(init) } + }}) +} + +enum InitKind { + Normal, + Zeroing, +} + +fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKind { + let Some((dotdot, expr)) = rest else { + return InitKind::Normal; + }; + match &expr { + Expr::Call(ExprCall { func, args, .. }) if args.is_empty() => match &**func { + Expr::Path(ExprPath { + attrs, + qself: None, + path: + Path { + leading_colon: None, + segments, + }, + }) if attrs.is_empty() + && segments.len() == 2 + && segments[0].ident == "Zeroable" + && segments[0].arguments.is_none() + && segments[1].ident == "init_zeroed" + && segments[1].arguments.is_none() => + { + return InitKind::Zeroing; + } + _ => {} + }, + _ => {} + } + dcx.error( + dotdot.span().join(expr.span()).unwrap_or(expr.span()), + "expected nothing or `..Zeroable::init_zeroed()`.", + ); + InitKind::Normal +} + +/// Generate the code that initializes the fields of the struct using the initializers in `field`. +fn init_fields( + fields: &Punctuated, + pinned: bool, + data: &Ident, + slot: &Ident, +) -> TokenStream { + let mut guards = vec![]; + let mut guard_attrs = vec![]; + let mut res = TokenStream::new(); + for InitializerField { attrs, kind } in fields { + let cfgs = { + let mut cfgs = attrs.clone(); + cfgs.retain(|attr| attr.path().is_ident("cfg")); + cfgs + }; + + let ident = match kind { + InitializerKind::Value { ident, .. } => ident, + InitializerKind::Init { ident, .. } => ident, + InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; + res.extend(quote! { + #(#attrs)* + { + #(#stmt)* + } + }); + continue; + } + }; + + let slot = if pinned { + quote! { + // SAFETY: + // - `slot` is valid and properly aligned. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { #data.#ident(#slot) }) + } + } else { + quote! { + // For `init!()` macro, everything is unpinned. + // SAFETY: + // - `&raw mut (*slot).#ident` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { + ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( + &raw mut (*#slot).#ident + ) + }) + } + }; + + // `mixed_site` ensures that the guard is not accessible to the user-controlled code. + let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + + let init = match kind { + InitializerKind::Value { ident, value } => { + let value = value + .as_ref() + .map(|(_, value)| quote!(#value)) + .unwrap_or_else(|| quote!(#ident)); + + quote! { + #(#attrs)* + let mut #guard = #slot.write(#value); + + } + } + InitializerKind::Init { value, .. } => { + quote! { + #(#attrs)* + let mut #guard = #slot.init(#value)?; + } + } + InitializerKind::Code { .. } => unreachable!(), + }; + + res.extend(quote! { + #init + + #(#cfgs)* + // Allow `non_snake_case` since the same warning is going to be reported for the struct + // field. + #[allow(unused_variables, non_snake_case)] + let #ident = #guard.let_binding(); + }); + + guards.push(guard); + guard_attrs.push(cfgs); + } + quote! { + #res + // If execution reaches this point, all fields have been initialized. Therefore we can now + // dismiss the guards by forgetting them. + #( + #(#guard_attrs)* + ::core::mem::forget(#guards); + )* + } +} + +/// Generate the check for ensuring that every field has been initialized and aligned. +fn make_field_check( + fields: &Punctuated, + init_kind: InitKind, + path: &Path, +) -> TokenStream { + let field_attrs: Vec<_> = fields + .iter() + .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let zeroing_trailer = match init_kind { + InitKind::Normal => None, + InitKind::Zeroing => Some(quote! { + ..::core::mem::zeroed() + }), + }; + quote! { + #[allow(unreachable_code)] + // We use unreachable code to perform field checks. They're still checked by the compiler. + // SAFETY: this code is never executed. + let _ = || unsafe { + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #( + #(#field_attrs)* + let _ = &(*slot).#field_name; + )* + + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #( + #(#field_attrs)* + #field_name: loop {}, + )* + #zeroing_trailer + }) + }; + } +} + +impl Parse for Initializer { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let attrs = input.call(Attribute::parse_outer)?; + let this = input.peek(Token![&]).then(|| input.parse()).transpose()?; + let path = input.parse()?; + let content; + let brace_token = braced!(content in input); + let mut fields = Punctuated::new(); + loop { + let lh = content.lookahead1(); + if lh.peek(End) || lh.peek(Token![..]) { + break; + } else if lh.peek(Ident) || lh.peek(Token![_]) || lh.peek(Token![#]) { + fields.push_value(content.parse()?); + let lh = content.lookahead1(); + if lh.peek(End) { + break; + } else if lh.peek(Token![,]) { + fields.push_punct(content.parse()?); + } else { + return Err(lh.error()); + } + } else { + return Err(lh.error()); + } + } + let rest = content + .peek(Token![..]) + .then(|| Ok::<_, syn::Error>((content.parse()?, content.parse()?))) + .transpose()?; + let error = input + .peek(Token![?]) + .then(|| Ok::<_, syn::Error>((input.parse()?, input.parse()?))) + .transpose()?; + let attrs = attrs + .into_iter() + .map(|a| { + if a.path().is_ident("default_error") { + a.parse_args::() + .map(InitializerAttribute::DefaultError) + } else { + Err(syn::Error::new_spanned(a, "unknown initializer attribute")) + } + }) + .collect::, _>>()?; + Ok(Self { + attrs, + this, + path, + brace_token, + fields, + rest, + error, + }) + } +} + +impl Parse for DefaultErrorAttribute { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + Ok(Self { ty: input.parse()? }) + } +} + +impl Parse for This { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + Ok(Self { + _and_token: input.parse()?, + ident: input.parse()?, + _in_token: input.parse()?, + }) + } +} + +impl Parse for InitializerField { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let attrs = input.call(Attribute::parse_outer)?; + Ok(Self { + attrs, + kind: input.parse()?, + }) + } +} + +impl Parse for InitializerKind { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let lh = input.lookahead1(); + if lh.peek(Token![_]) { + Ok(Self::Code { + _underscore_token: input.parse()?, + _colon_token: input.parse()?, + block: input.parse()?, + }) + } else if lh.peek(Ident) { + let ident = input.parse()?; + let lh = input.lookahead1(); + if lh.peek(Token![<-]) { + Ok(Self::Init { + ident, + _left_arrow_token: input.parse()?, + value: input.parse()?, + }) + } else if lh.peek(Token![:]) { + Ok(Self::Value { + ident, + value: Some((input.parse()?, input.parse()?)), + }) + } else if lh.peek(Token![,]) || lh.peek(End) { + Ok(Self::Value { ident, value: None }) + } else { + Err(lh.error()) + } + } else { + Err(lh.error()) + } + } +} diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs new file mode 100644 index 000000000..60d5093f3 --- /dev/null +++ b/rust/pin-init/internal/src/lib.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! `pin-init` proc macros. + +// Documentation is done in the pin-init crate instead. +#![allow(missing_docs)] + +use proc_macro::TokenStream; +use syn::parse_macro_input; + +use crate::diagnostics::DiagCtxt; + +mod diagnostics; +mod init; +mod pin_data; +mod pinned_drop; +mod zeroable; + +#[proc_macro_attribute] +pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args); + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| pin_data::pin_data(args, input, dcx)).into() +} + +#[proc_macro_attribute] +pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args); + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into() +} + +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| zeroable::derive(input, dcx)).into() +} + +#[proc_macro_derive(MaybeZeroable)] +pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| zeroable::maybe_derive(input, dcx)).into() +} +#[proc_macro] +pub fn init(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), false, dcx)) + .into() +} + +#[proc_macro] +pub fn pin_init(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), true, dcx)).into() +} diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs new file mode 100644 index 000000000..ff194d275 --- /dev/null +++ b/rust/pin-init/internal/src/pin_data.rs @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::{format_ident, quote, ToTokens}; +use syn::{ + parse::{End, Nothing, Parse}, + parse_quote, parse_quote_spanned, + spanned::Spanned, + visit_mut::VisitMut, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, +}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) mod kw { + syn::custom_keyword!(PinnedDrop); +} + +pub(crate) enum Args { + Nothing(Nothing), + #[allow(dead_code)] + PinnedDrop(kw::PinnedDrop), +} + +impl Parse for Args { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let lh = input.lookahead1(); + if lh.peek(End) { + input.parse().map(Self::Nothing) + } else if lh.peek(kw::PinnedDrop) { + input.parse().map(Self::PinnedDrop) + } else { + Err(lh.error()) + } + } +} + +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + +struct FieldInfo<'a> { + field: &'a Field, + pinned: bool, +} + +pub(crate) fn pin_data( + args: Args, + input: Item, + dcx: &mut DiagCtxt, +) -> Result { + let mut struct_ = match input { + Item::Struct(struct_) => struct_, + Item::Enum(enum_) => { + return Err(dcx.error( + enum_.enum_token, + "`#[pin_data]` only supports structs for now", + )); + } + Item::Union(union) => { + return Err(dcx.error( + union.union_token, + "`#[pin_data]` only supports structs for now", + )); + } + rest => { + return Err(dcx.error( + rest, + "`#[pin_data]` can only be applied to struct, enum and union definitions", + )); + } + }; + + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + + // The generics might contain the `Self` type. Since this macro will define a new type with the + // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed + // to this struct definition. Therefore we have to replace `Self` with the concrete name. + let mut replacer = { + let name = &struct_.ident; + let (_, ty_generics, _) = struct_.generics.split_for_impl(); + SelfReplacer(parse_quote!(#name #ty_generics)) + }; + replacer.visit_generics_mut(&mut struct_.generics); + replacer.visit_fields_mut(&mut struct_.fields); + + let fields: Vec> = struct_ + .fields + .iter_mut() + .map(|field| { + let len = field.attrs.len(); + field.attrs.retain(|a| !a.path().is_ident("pin")); + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } + + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); + + FieldInfo { + field: &*field, + pinned: pinned_count != 0, + } + }) + .collect(); + + for field in &fields { + let ident = field.field.ident.as_ref().unwrap(); + + if !field.pinned && is_phantom_pinned(&field.field.ty) { + dcx.warn( + field.field, + format!( + "The field `{ident}` of type `PhantomPinned` only has an effect \ + if it has the `#[pin]` attribute", + ), + ); + } + } + + let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields); + let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args); + let projections = + generate_projections(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + let the_pin_data = + generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + + Ok(quote! { + #struct_ + #projections + // We put the rest into this const item, because it then will not be accessible to anything + // outside. + const _: () = { + #the_pin_data + #unpin_impl + #drop_impl + }; + }) +} + +fn is_phantom_pinned(ty: &Type) -> bool { + match ty { + Type::Path(TypePath { qself: None, path }) => { + // Cannot possibly refer to `PhantomPinned` (except alias, but that's on the user). + if path.segments.len() > 3 { + return false; + } + // If there is a `::`, then the path needs to be `::core::marker::PhantomPinned` or + // `::std::marker::PhantomPinned`. + if path.leading_colon.is_some() && path.segments.len() != 3 { + return false; + } + let expected: Vec<&[&str]> = vec![&["PhantomPinned"], &["marker"], &["core", "std"]]; + for (actual, expected) in path.segments.iter().rev().zip(expected) { + if !actual.arguments.is_empty() || expected.iter().all(|e| actual.ident != e) { + return false; + } + } + true + } + _ => false, + } +} + +fn generate_unpin_impl( + ident: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (_, ty_generics, _) = generics.split_for_impl(); + let mut generics_with_pin_lt = generics.clone(); + generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); + generics_with_pin_lt.make_where_clause(); + let ( + impl_generics_with_pin_lt, + ty_generics_with_pin_lt, + Some(WhereClause { + where_token, + predicates, + }), + ) = generics_with_pin_lt.split_for_impl() + else { + unreachable!() + }; + let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { + let ident = f.field.ident.as_ref().unwrap(); + let ty = &f.field.ty; + quote!( + #ident: #ty + ) + }); + quote! { + // This struct will be used for the unpin analysis. It is needed, because only structurally + // pinned fields are relevant whether the struct should implement `Unpin`. + #[allow( + dead_code, // The fields below are never used. + non_snake_case // The warning will be emitted on the struct definition. + )] + struct __Unpin #generics_with_pin_lt + #where_token + #predicates + { + __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>, + __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>, + #(#pinned_fields),* + } + + #[doc(hidden)] + impl #impl_generics_with_pin_lt ::core::marker::Unpin for #ident #ty_generics + #where_token + __Unpin #ty_generics_with_pin_lt: ::core::marker::Unpin, + #predicates + {} + } +} + +fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenStream { + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let has_pinned_drop = matches!(args, Args::PinnedDrop(_)); + // We need to disallow normal `Drop` implementation, the exact behavior depends on whether + // `PinnedDrop` was specified in `args`. + if has_pinned_drop { + // When `PinnedDrop` was specified we just implement `Drop` and delegate. + quote! { + impl #impl_generics ::core::ops::Drop for #ident #ty_generics + #whr + { + fn drop(&mut self) { + // SAFETY: Since this is a destructor, `self` will not move after this function + // terminates, since it is inaccessible. + let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; + // SAFETY: Since this is a drop function, we can create this token to call the + // pinned destructor of this type. + let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() }; + ::pin_init::PinnedDrop::drop(pinned, token); + } + } + } + } else { + // When no `PinnedDrop` was specified, then we have to prevent implementing drop. + quote! { + // We prevent this by creating a trait that will be implemented for all types implementing + // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, + // if it also implements `Drop` + trait MustNotImplDrop {} + impl MustNotImplDrop for T {} + impl #impl_generics MustNotImplDrop for #ident #ty_generics + #whr + {} + // We also take care to prevent users from writing a useless `PinnedDrop` implementation. + // They might implement `PinnedDrop` correctly for the struct, but forget to give + // `PinnedDrop` as the parameter to `#[pin_data]`. + trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} + impl + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} + impl #impl_generics + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for #ident #ty_generics + #whr + {} + } + } +} + +fn generate_projections( + vis: &Visibility, + ident: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (impl_generics, ty_generics, _) = generics.split_for_impl(); + let mut generics_with_pin_lt = generics.clone(); + generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); + let (_, ty_generics_with_pin_lt, whr) = generics_with_pin_lt.split_for_impl(); + let projection = format_ident!("{ident}Projection"); + let this = format_ident!("this"); + + let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields + .iter() + .map(|field| { + let Field { vis, ident, ty, .. } = &field.field; + + let ident = ident + .as_ref() + .expect("only structs with named fields are supported"); + if field.pinned { + ( + quote!( + #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, + ), + quote!( + // SAFETY: this field is structurally pinned. + #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, + ), + ) + } else { + ( + quote!( + #vis #ident: &'__pin mut #ty, + ), + quote!( + #ident: &mut #this.#ident, + ), + ) + } + }) + .collect(); + let structurally_pinned_fields_docs = fields + .iter() + .filter(|f| f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + let not_structurally_pinned_fields_docs = fields + .iter() + .filter(|f| !f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + let docs = format!(" Pin-projections of [`{ident}`]"); + quote! { + #[doc = #docs] + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] + #[doc(hidden)] + #vis struct #projection #generics_with_pin_lt + #whr + { + #(#fields_decl)* + ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, + } + + impl #impl_generics #ident #ty_generics + #whr + { + /// Pin-projects all fields of `Self`. + /// + /// These fields are structurally pinned: + #(#[doc = #structurally_pinned_fields_docs])* + /// + /// These fields are **not** structurally pinned: + #(#[doc = #not_structurally_pinned_fields_docs])* + #[inline] + #vis fn project<'__pin>( + self: ::core::pin::Pin<&'__pin mut Self>, + ) -> #projection #ty_generics_with_pin_lt { + // SAFETY: we only give access to `&mut` for fields not structurally pinned. + let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; + #projection { + #(#fields_proj)* + ___pin_phantom_data: ::core::marker::PhantomData, + } + } + } + } +} + +fn generate_the_pin_data( + vis: &Visibility, + struct_name: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + + // For every field, we create an initializing projection function according to its projection + // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be + // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with + // `Unpinned` which allows initialization via `Init`. + let field_accessors = fields + .iter() + .map(|f| { + let Field { vis, ident, ty, .. } = f.field; + + let field_name = ident + .as_ref() + .expect("only structs with named fields are supported"); + let pin_marker = if f.pinned { + quote!(Pinned) + } else { + quote!(Unpinned) + }; + quote! { + /// # Safety + /// + /// - `slot` is valid and properly aligned. + /// - `(*slot).#field_name` is properly aligned. + /// - `(*slot).#field_name` points to uninitialized and exclusively accessed + /// memory. + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(non_snake_case)] + #[inline(always)] + #vis unsafe fn #field_name( + self, + slot: *mut #struct_name #ty_generics, + ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> { + // SAFETY: + // - If `#pin_marker` is `Pinned`, the corresponding field is structurally + // pinned. + // - Other safety requirements follows the safety requirement. + unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) } + } + } + }) + .collect::(); + quote! { + // We declare this struct which will host all of the projection function for our type. It + // will be invariant over all generic parameters which are inherited from the struct. + #[doc(hidden)] + #vis struct __ThePinData #generics + #whr + { + __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>, + } + + impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics + #whr + { + #[inline] + fn clone(&self) -> Self { *self } + } + + impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics + #whr + {} + + #[allow(dead_code)] // Some functions might never be used and private. + impl #impl_generics __ThePinData #ty_generics + #whr + { + /// Type inference helper function. + #[inline(always)] + #vis fn __make_closure<__F, __E>(self, f: __F) -> __F + where + __F: FnOnce(*mut #struct_name #ty_generics) -> + ::core::result::Result<::pin_init::__internal::InitOk, __E>, + { + f + } + + #field_accessors + } + + // SAFETY: We have added the correct projection functions above to `__ThePinData` and + // we also use the least restrictive generics possible. + unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics + #whr + { + type PinData = __ThePinData #ty_generics; + + #[inline] + unsafe fn __pin_data() -> Self::PinData { + __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } + } + } + } +} + +struct SelfReplacer(PathSegment); + +impl VisitMut for SelfReplacer { + fn visit_path_mut(&mut self, i: &mut syn::Path) { + if i.is_ident("Self") { + let span = i.span(); + let seg = &self.0; + *i = parse_quote_spanned!(span=> #seg); + } else { + syn::visit_mut::visit_path_mut(self, i); + } + } + + fn visit_path_segment_mut(&mut self, seg: &mut PathSegment) { + if seg.ident == "Self" { + let span = seg.span(); + let this = &self.0; + *seg = parse_quote_spanned!(span=> #this); + } else { + syn::visit_mut::visit_path_segment_mut(self, seg); + } + } + + fn visit_item_mut(&mut self, _: &mut Item) { + // Do not descend into items, since items reset/change what `Self` refers to. + } +} diff --git a/rust/pin-init/internal/src/pinned_drop.rs b/rust/pin-init/internal/src/pinned_drop.rs new file mode 100644 index 000000000..a20ac314c --- /dev/null +++ b/rust/pin-init/internal/src/pinned_drop.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse::Nothing, parse_quote, spanned::Spanned, ImplItem, ItemImpl, Token}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) fn pinned_drop( + _args: Nothing, + mut input: ItemImpl, + dcx: &mut DiagCtxt, +) -> Result { + if let Some(unsafety) = input.unsafety { + dcx.error(unsafety, "implementing `PinnedDrop` is safe"); + } + input.unsafety = Some(Token![unsafe](input.impl_token.span)); + match &mut input.trait_ { + Some((not, path, _for)) => { + if let Some(not) = not { + dcx.error(not, "cannot implement `!PinnedDrop`"); + } + for (seg, expected) in path + .segments + .iter() + .rev() + .zip(["PinnedDrop", "pin_init", ""]) + { + if expected.is_empty() || seg.ident != expected { + dcx.error(seg, "bad import path for `PinnedDrop`"); + } + if !seg.arguments.is_none() { + dcx.error(&seg.arguments, "unexpected arguments for `PinnedDrop` path"); + } + } + *path = parse_quote!(::pin_init::PinnedDrop); + } + None => { + let span = input + .impl_token + .span + .join(input.self_ty.span()) + .unwrap_or(input.impl_token.span); + dcx.error( + span, + "expected `impl ... PinnedDrop for ...`, got inherent impl", + ); + } + } + for item in &mut input.items { + if let ImplItem::Fn(fn_item) = item { + if fn_item.sig.ident == "drop" { + fn_item + .sig + .inputs + .push(parse_quote!(_: ::pin_init::__internal::OnlyCallFromDrop)); + } + } + } + Ok(quote!(#input)) +} diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs new file mode 100644 index 000000000..b11feaeb1 --- /dev/null +++ b/rust/pin-init/internal/src/zeroable.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse_quote, Data, DeriveInput, Field, Fields}; + +use crate::{diagnostics::ErrorGuaranteed, DiagCtxt}; + +pub(crate) fn derive( + input: DeriveInput, + dcx: &mut DiagCtxt, +) -> Result { + let fields = match input.data { + Data::Struct(data_struct) => data_struct.fields, + Data::Union(data_union) => Fields::Named(data_union.fields), + Data::Enum(data_enum) => { + return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum")); + } + }; + let name = input.ident; + let mut generics = input.generics; + for param in generics.type_params_mut() { + param.bounds.insert(0, parse_quote!(::pin_init::Zeroable)); + } + let (impl_gen, ty_gen, whr) = generics.split_for_impl(); + let field_type = fields.iter().map(|field| &field.ty); + Ok(quote! { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen + #whr + {} + const _: () = { + fn assert_zeroable() {} + fn ensure_zeroable #impl_gen () + #whr + { + #( + assert_zeroable::<#field_type>(); + )* + } + }; + }) +} + +pub(crate) fn maybe_derive( + input: DeriveInput, + dcx: &mut DiagCtxt, +) -> Result { + let fields = match input.data { + Data::Struct(data_struct) => data_struct.fields, + Data::Union(data_union) => Fields::Named(data_union.fields), + Data::Enum(data_enum) => { + return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum")); + } + }; + let name = input.ident; + let mut generics = input.generics; + for param in generics.type_params_mut() { + param.bounds.insert(0, parse_quote!(::pin_init::Zeroable)); + } + for Field { ty, .. } in fields { + generics + .make_where_clause() + .predicates + // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` + // feature . + .push(parse_quote!(#ty: for<'__dummy> ::pin_init::Zeroable)); + } + let (impl_gen, ty_gen, whr) = generics.split_for_impl(); + Ok(quote! { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen + #whr + {} + }) +} diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs new file mode 100644 index 000000000..8e9fd18b9 --- /dev/null +++ b/rust/pin-init/src/__internal.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module contains library internal items. +//! +//! These items must not be used outside of this crate and the pin-init-internal crate located at +//! `../internal`. + +use super::*; + +/// Zero-sized type used to mark a type as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// See the [nomicon] for what subtyping is. See also [this table]. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html +/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html +/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns +#[repr(transparent)] +pub struct PhantomInvariant(PhantomData T>); + +impl Clone for PhantomInvariant { + #[inline(always)] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for PhantomInvariant {} + +impl Default for PhantomInvariant { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl PhantomInvariant { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +/// Zero-sized type used to mark a lifetime as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html +#[repr(transparent)] +#[derive(Clone, Copy, Default)] +pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); + +impl PhantomInvariantLifetime<'_> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomInvariant::new()) + } +} + +/// Token type to signify successful initialization. +/// +/// Can only be constructed via the unsafe [`Self::new`] function. The initializer macros use this +/// token type to prevent returning `Ok` from an initializer without initializing all fields. +pub struct InitOk(()); + +impl InitOk { + /// Creates a new token. + /// + /// # Safety + /// + /// This function may only be called from the `init!` macro in `../internal/src/init.rs`. + #[inline(always)] + pub unsafe fn new() -> Self { + Self(()) + } +} + +/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate +/// the pin projections within the initializers. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasPinData { + type PinData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __pin_data() -> Self::PinData; +} + +/// This trait is automatically implemented for every type. It aims to provide the same type +/// inference help as `HasPinData`. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasInitData { + type InitData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __init_data() -> Self::InitData; +} + +pub struct AllData(PhantomInvariant); + +impl Clone for AllData { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for AllData {} + +impl AllData { + /// Type inference helper function. + #[inline(always)] + pub fn __make_closure(self, f: F) -> F + where + F: FnOnce(*mut T) -> Result, + { + f + } +} + +// SAFETY: TODO. +unsafe impl HasInitData for T { + type InitData = AllData; + + #[inline] + unsafe fn __init_data() -> Self::InitData { + AllData(PhantomInvariant::new()) + } +} + +/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. +/// +/// # Invariants +/// +/// If `self.is_init` is true, then `self.value` is initialized. +/// +/// [`stack_pin_init`]: crate::stack_pin_init +pub struct StackInit { + value: MaybeUninit, + is_init: bool, +} + +impl Drop for StackInit { + #[inline] + fn drop(&mut self) { + if self.is_init { + // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is + // true, `self.value` is initialized. + unsafe { self.value.assume_init_drop() }; + } + } +} + +impl StackInit { + /// Creates a new [`StackInit`] that is uninitialized. Use [`stack_pin_init`] instead of this + /// primitive. + /// + /// [`stack_pin_init`]: crate::stack_pin_init + #[inline] + pub fn uninit() -> Self { + Self { + value: MaybeUninit::uninit(), + is_init: false, + } + } + + /// Initializes the contents and returns the result. + #[inline] + pub fn init(self: Pin<&mut Self>, init: impl PinInit) -> Result, E> { + // SAFETY: We never move out of `this`. + let this = unsafe { Pin::into_inner_unchecked(self) }; + // The value is currently initialized, so it needs to be dropped before we can reuse + // the memory (this is a safety guarantee of `Pin`). + if this.is_init { + this.is_init = false; + // SAFETY: `this.is_init` was true and therefore `this.value` is initialized. + unsafe { this.value.assume_init_drop() }; + } + // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. + unsafe { init.__init(this.value.as_mut_ptr())? }; + // INVARIANT: `this.value` is initialized above. + this.is_init = true; + // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. + Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) }) + } +} + +#[test] +#[cfg(feature = "std")] +fn stack_init_reuse() { + use ::std::{borrow::ToOwned, println, string::String}; + use core::pin::pin; + + #[derive(Debug)] + struct Foo { + a: usize, + b: String, + } + let mut slot: Pin<&mut StackInit> = pin!(StackInit::uninit()); + let value: Result, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 42, + b: "Hello".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); + let value: Result, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 24, + b: "world!".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); +} + +// Marker types that determines type of `DropGuard`'s let bindings. +pub struct Pinned; +pub struct Unpinned; + +/// Represent an uninitialized field. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory. +/// - If `P` is `Pinned`, then `ptr` is structurally pinned. +pub struct Slot { + ptr: *mut T, + _phantom: PhantomData

, +} + +impl Slot { + /// # Safety + /// + /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed + /// memory. + /// - If `P` is `Pinned`, then `ptr` is structurally pinned. + #[inline(always)] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + _phantom: PhantomData, + } + } + + /// Initialize the field by value. + #[inline(always)] + pub fn write(self, value: T) -> DropGuard + where + T: Sized, + { + // SAFETY: `self.ptr` is a valid and aligned pointer for write. + unsafe { self.ptr.write(value) } + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + // - If `P` is `Pinned`, `self.ptr` is pinned. + unsafe { DropGuard::new(self.ptr) } + } +} + +impl Slot { + /// Initialize the field. + #[inline(always)] + pub fn init(self, init: impl Init) -> Result, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `slot`; + // also `self` is consumed so it cannot be touched further. + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +impl Slot { + /// Initialize the field. + #[inline(always)] + pub fn init(self, init: impl PinInit) -> Result, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and pinned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +/// When a value of this type is dropped, it drops a `T`. +/// +/// Can be forgotten to prevent the drop. +/// +/// # Invariants +/// +/// - `ptr` is valid and properly aligned. +/// - `*ptr` is initialized and owned by this guard. +/// - if `P` is `Pinned`, `ptr` is pinned. +pub struct DropGuard { + ptr: *mut T, + phantom: PhantomData

, +} + +impl DropGuard { + /// Creates a drop guard and transfer the ownership of the pointer content. + /// + /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`]. + /// + /// # Safety + /// + /// - `ptr` is valid and properly aligned. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. + /// - if `P` is `Pinned`, `ptr` is pinned. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: By safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } +} + +impl DropGuard { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } +} + +impl DropGuard { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&mut T> { + // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and + // pinned per type invariant. + unsafe { Pin::new_unchecked(&mut *self.ptr) } + } +} + +impl Drop for DropGuard { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. + unsafe { ptr::drop_in_place(self.ptr) } + } +} + +/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely +/// created struct. This is needed, because the `drop` function is safe, but should not be called +/// manually. +pub struct OnlyCallFromDrop(()); + +impl OnlyCallFromDrop { + /// # Safety + /// + /// This function should only be called from the [`Drop::drop`] function and only be used to + /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type. + pub unsafe fn new() -> Self { + Self(()) + } +} + +/// Initializer that always fails. +/// +/// Used by [`assert_pinned!`]. +/// +/// [`assert_pinned!`]: crate::assert_pinned +pub struct AlwaysFail { + _t: PhantomData, +} + +impl AlwaysFail { + /// Creates a new initializer that always fails. + #[inline] + pub fn new() -> Self { + Self { _t: PhantomData } + } +} + +impl Default for AlwaysFail { + #[inline] + fn default() -> Self { + Self::new() + } +} + +// SAFETY: `__init` always fails, which is always okay. +unsafe impl PinInit for AlwaysFail { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { + Err(()) + } +} diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs new file mode 100644 index 000000000..471652e86 --- /dev/null +++ b/rust/pin-init/src/alloc.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(all(feature = "alloc", not(feature = "std")))] +use alloc::{boxed::Box, sync::Arc}; +#[cfg(feature = "alloc")] +use core::alloc::AllocError; +use core::{mem::MaybeUninit, pin::Pin}; +#[cfg(feature = "std")] +use std::sync::Arc; + +#[cfg(not(feature = "alloc"))] +type AllocError = core::convert::Infallible; + +use crate::{ + init_from_closure, pin_init_from_closure, InPlaceWrite, Init, PinInit, ZeroableOption, +}; + +pub extern crate alloc; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// ). +unsafe impl ZeroableOption for Box {} + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit: Sized { + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] + fn pin_init(init: impl PinInit) -> Result, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_pin_init(init) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init(init: impl Init) -> Result + where + E: From; + + /// Use the given initializer to in-place initialize a `T`. + #[inline] + fn init(init: impl Init) -> Result { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_init(init) + } +} + +#[cfg(feature = "alloc")] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::try_new_uninit()? + }; +} +#[cfg(all(feature = "std", not(feature = "alloc")))] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::new_uninit() + }; +} + +impl InPlaceInit for Box { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + try_new_uninit!(Box).write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + try_new_uninit!(Box).write_init(init) + } +} + +impl InPlaceInit for Arc { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized and this is the only `Arc` to that data. + Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} + +impl InPlaceWrite for Box> { + type Initialized = Box; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs new file mode 100644 index 000000000..f1463be94 --- /dev/null +++ b/rust/pin-init/src/lib.rs @@ -0,0 +1,1797 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Library to safely and fallibly initialize pinned `struct`s using in-place constructors. +//! +//! [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack +//! overflow. +//! +//! This library's main use-case is in [Rust-for-Linux]. Although this version can be used +//! standalone. +//! +//! There are cases when you want to in-place initialize a struct. For example when it is very big +//! and moving it from the stack is not an option, because it is bigger than the stack itself. +//! Another reason would be that you need the address of the object to initialize it. This stands +//! in direct conflict with Rust's normal process of first initializing an object and then moving +//! it into it's final memory location. For more information, see +//! . +//! +//! This library allows you to do in-place initialization safely. +//! +//! ## Nightly Needed for `alloc` feature +//! +//! This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +//! enabled and thus this feature can only be used with a nightly compiler. When enabling the +//! `alloc` feature, the user will be required to activate `allocator_api` as well. +//! +//! [`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html +//! +//! The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +//! However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +//! will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +//! mode. +//! +//! ## Nightly needed for `unsafe-pinned` feature +//! +//! This feature enables the `Wrapper` implementation on the unstable `core::pin::UnsafePinned` type. +//! This requires the [`unsafe_pinned` unstable feature](https://github.com/rust-lang/rust/issues/125735) +//! and therefore a nightly compiler. Note that this feature is not enabled by default. +//! +//! # Overview +//! +//! To initialize a `struct` with an in-place constructor you will need two things: +//! - an in-place constructor, +//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], +//! [`Box`] or any other smart pointer that supports this library). +//! +//! To get an in-place constructor there are generally three options: +//! - directly creating an in-place constructor using the [`pin_init!`] macro, +//! - a custom function/macro returning an in-place constructor provided by someone else, +//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! +//! Aside from pinned initialization, this library also supports in-place construction without +//! pinning, the macros/types/functions are generally named like the pinned variants without the +//! `pin_` prefix. +//! +//! # Examples +//! +//! Throughout the examples we will often make use of the `CMutex` type which can be found in +//! `../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +//! the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +//! requires it to be pinned to be locked and thus is a prime candidate for using this library. +//! +//! ## Using the [`pin_init!`] macro +//! +//! If you want to use [`PinInit`], then you will have to annotate your `struct` with +//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via +//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::pin::Pin; +//! use pin_init::{pin_data, pin_init, InPlaceInit}; +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! a: CMutex, +//! b: u32, +//! } +//! +//! let foo = pin_init!(Foo { +//! a <- CMutex::new(42), +//! b: 24, +//! }); +//! # let _ = Box::pin_init(foo); +//! ``` +//! +//! `foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like +//! (or just the stack) to actually initialize a `Foo`: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::{alloc::AllocError, pin::Pin}; +//! # use pin_init::*; +//! # +//! # #[pin_data] +//! # struct Foo { +//! # #[pin] +//! # a: CMutex, +//! # b: u32, +//! # } +//! # +//! # let foo = pin_init!(Foo { +//! # a <- CMutex::new(42), +//! # b: 24, +//! # }); +//! let foo: Result>, AllocError> = Box::pin_init(foo); +//! ``` +//! +//! For more information see the [`pin_init!`] macro. +//! +//! ## Using a custom function/macro that returns an initializer +//! +//! Many types that use this library supply a function/macro that returns an initializer, because +//! the above method only works for types where you can access the fields. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use pin_init::*; +//! # use std::sync::Arc; +//! # use core::pin::Pin; +//! let mtx: Result>>, _> = Arc::pin_init(CMutex::new(42)); +//! ``` +//! +//! To declare an init macro/function you just return an [`impl PinInit`]: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # use pin_init::*; +//! # #[path = "../examples/error.rs"] mod error; use error::Error; +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! #[pin_data] +//! struct DriverData { +//! #[pin] +//! status: CMutex, +//! buffer: Box<[u8; 1_000_000]>, +//! } +//! +//! impl DriverData { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! status <- CMutex::new(0), +//! buffer: Box::init(pin_init::init_zeroed())?, +//! }? Error) +//! } +//! } +//! ``` +//! +//! ## Manual creation of an initializer +//! +//! Often when working with primitives the previous approaches are not sufficient. That is where +//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +//! [`impl PinInit`] directly from a closure. Of course you have to ensure that the closure +//! actually does the initialization in the correct way. Here are the things to look out for +//! (we are calling the parameter to the closure `slot`): +//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so +//! `slot` now contains a valid bit pattern for the type `T`, +//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so +//! you need to take care to clean up anything if your initialization fails mid-way, +//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of +//! `slot` gets called. +//! +//! ```rust +//! # #![feature(extern_types)] +//! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +//! use core::{ +//! marker::PhantomPinned, +//! cell::UnsafeCell, +//! pin::Pin, +//! mem::MaybeUninit, +//! }; +//! mod bindings { +//! #[repr(C)] +//! pub struct foo { +//! /* fields from C ... */ +//! } +//! extern "C" { +//! pub fn init_foo(ptr: *mut foo); +//! pub fn destroy_foo(ptr: *mut foo); +//! #[must_use = "you must check the error return code"] +//! pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; +//! } +//! } +//! +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! _p: PhantomPinned, +//! #[pin] +//! foo: UnsafeCell>, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = &raw mut (*slot).foo; +//! let foo = UnsafeCell::raw_get(foo).cast::(); +//! +//! // Initialize the `foo` +//! bindings::init_foo(foo); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(foo, flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(foo); +//! Err(err) +//! } else { +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! } +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get().cast::()) }; +//! } +//! } +//! ``` +//! +//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +//! the `kernel` crate. The [`sync`] module is a good starting point. +//! +//! [`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [structurally pinned fields]: +//! https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning +//! [stack]: crate::stack_pin_init +#![cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#![cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#![cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#![cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +//! [`impl PinInit`]: crate::PinInit +//! [`impl PinInit`]: crate::PinInit +//! [`impl Init`]: crate::Init +//! [Rust-for-Linux]: https://rust-for-linux.com/ + +#![forbid(missing_docs, unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![cfg_attr( + all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED), + feature(unsafe_pinned) +)] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), allow(internal_features))] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), feature(rustdoc_internals))] + +use core::{ + cell::UnsafeCell, + convert::Infallible, + marker::PhantomData, + mem::MaybeUninit, + num::*, + pin::Pin, + ptr::{self, NonNull}, +}; + +// This is used by doc-tests -- the proc-macros expand to `::pin_init::...` and without this the +// doc-tests wouldn't have an extern crate named `pin_init`. +#[allow(unused_extern_crates)] +extern crate self as pin_init; + +#[doc(hidden)] +pub mod __internal; + +#[cfg(any(feature = "std", feature = "alloc"))] +mod alloc; +#[cfg(any(feature = "std", feature = "alloc"))] +pub use alloc::InPlaceInit; + +/// Used to specify the pinning information of the fields of a struct. +/// +/// This is somewhat similar in purpose as +/// [pin-project-lite](https://crates.io/crates/pin-project-lite). +/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each +/// field you want to structurally pin. +/// +/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, +/// then `#[pin]` directs the type of initializer that is required. +/// +/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this +/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with +/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// use pin_init::pin_data; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// } +/// ``` +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pin_data; + +/// Used to implement `PinnedDrop` safely. +/// +/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pinned_drop; + +/// Derives the [`Zeroable`] trait for the given `struct` or `union`. +/// +/// This can only be used for `struct`s/`union`s where every field implements the [`Zeroable`] +/// trait. +/// +/// # Examples +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub union SignCast { +/// signed: i64, +/// unsigned: u64, +/// } +/// ``` +pub use ::pin_init_internal::Zeroable; + +/// Derives the [`Zeroable`] trait for the given `struct` or `union` if all fields implement +/// [`Zeroable`]. +/// +/// Contrary to the derive macro named [`macro@Zeroable`], this one silently fails when a field +/// doesn't implement [`Zeroable`]. +/// +/// # Examples +/// +/// ``` +/// use pin_init::MaybeZeroable; +/// +/// // implements `Zeroable` +/// #[derive(MaybeZeroable)] +/// pub struct DriverData { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// +/// // does not implement `Zeroable` +/// #[derive(MaybeZeroable)] +/// pub struct DriverData2 { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// // this field doesn't implement `Zeroable` +/// other_data: &'static i32, +/// } +/// ``` +pub use ::pin_init_internal::MaybeZeroable; + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_pin_init!(let foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Bar { +/// x: 64, +/// }, +/// })); +/// let foo: Pin<&mut Foo> = foo; +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error +/// type, then use [`stack_try_pin_init!`]. +#[macro_export] +macro_rules! stack_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let Ok(mut $var) = $crate::__internal::StackInit::init($var, val); + }; +} + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// let foo = foo.unwrap(); +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo =? pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// println!("a: {}", &*foo.a.lock()); +/// # Ok::<_, Error>(()) +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the +/// `=` will propagate this error. +#[macro_export] +macro_rules! stack_try_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val); + }; + (let $var:ident $(: $t:ty)? =? $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val)?; + }; +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// The error type defaults to [`Infallible`]; if you need a different one, write `? Error` at the +/// end, after the struct initializer. +/// +/// The syntax is almost identical to that of a normal `struct` initializer: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// a: usize, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit { +/// let a = 42; +/// +/// let initializer = pin_init!(Foo { +/// a, +/// b: Bar { +/// x: 64, +/// }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// +/// Arbitrary Rust expressions can be used to set the value of a variable. +/// +/// The fields are initialized in the order that they appear in the initializer. So it is possible +/// to read already initialized fields using raw pointers. +/// +/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the +/// initializer. +/// +/// # Init-functions +/// +/// When working with this library it is often desired to let others construct your types without +/// giving access to all fields. This is where you would normally write a plain function `new` that +/// would return a new instance of your type. With this library that is also possible. However, +/// there are a few extra things to keep in mind. +/// +/// To create an initializer function, simply declare it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// impl Foo { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// a: 42, +/// b: Bar { +/// x: 64, +/// }, +/// }) +/// } +/// } +/// ``` +/// +/// Users of `Foo` can now create it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// let foo = Box::pin_init(Foo::new()); +/// ``` +/// +/// They can also easily embed it into their own `struct`s: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// #[pin_data] +/// struct FooContainer { +/// #[pin] +/// foo1: Foo, +/// #[pin] +/// foo2: Foo, +/// other: u32, +/// } +/// +/// impl FooContainer { +/// fn new(other: u32) -> impl PinInit { +/// pin_init!(Self { +/// foo1 <- Foo::new(), +/// foo2 <- Foo::new(), +/// other, +/// }) +/// } +/// } +/// ``` +/// +/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. +/// This signifies that the given field is initialized in-place. As with `struct` initializers, just +/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. +/// +/// # Syntax +/// +/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with +/// the following modifications is expected: +/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in +/// order to run arbitrary code. +/// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] +/// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::init_zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. +/// +/// For instance: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::marker::PhantomPinned; +/// #[pin_data] +/// #[derive(Zeroable)] +/// struct Buf { +/// // `ptr` points into `buf`. +/// ptr: *mut u8, +/// buf: [u8; 64], +/// #[pin] +/// pin: PhantomPinned, +/// } +/// +/// let init = pin_init!(&this in Buf { +/// buf: [0; 64], +/// // SAFETY: TODO. +/// ptr: unsafe { (&raw mut (*this.as_ptr()).buf).cast() }, +/// pin: PhantomPinned, +/// }); +/// let init = pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::init_zeroed() +/// }); +/// ``` +/// +/// [`NonNull`]: core::ptr::NonNull +pub use pin_init_internal::pin_init; + +/// Construct an in-place, fallible initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]; if you need a different one, write `? Error` +/// at the end, after the struct initializer. +/// +/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// This initializer is for initializing data in-place that might later be moved. If you want to +/// pin-initialize, use [`pin_init!`]. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::InPlaceInit; +/// use pin_init::{init, Init, init_zeroed}; +/// +/// struct BigBuf { +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init { +/// init!(Self { +/// small <- init_zeroed(), +/// }) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +pub use pin_init_internal::init; + +/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is +/// structurally pinned. +/// +/// # Examples +/// +/// This will succeed: +/// ``` +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// #[pin] +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// This will fail: +/// ```compile_fail +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To +/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can +/// only be used when the macro is invoked from a function body. +/// ``` +/// # use core::pin::Pin; +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// elem: T, +/// } +/// +/// impl Foo { +/// fn project_this(self: Pin<&mut Self>) -> Pin<&mut T> { +/// assert_pinned!(Foo, elem, T, inline); +/// +/// // SAFETY: The field is structurally pinned. +/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! assert_pinned { + ($ty:ty, $field:ident, $field_ty:ty, inline) => { + // SAFETY: This code is unreachable. + let _ = move |ptr: *mut $ty| unsafe { + let data = <$ty as $crate::__internal::HasPinData>::__pin_data(); + _ = data + .$field(ptr) + .init($crate::__internal::AlwaysFail::<$field_ty>::new()); + }; + }; + + ($ty:ty, $field:ident, $field_ty:ty) => { + const _: () = { + $crate::assert_pinned!($ty, $field, $field_ty, inline); + }; + }; +} + +/// A pin-initializer for the type `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`] or even the stack (see [`stack_pin_init!`]). +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. +/// +/// The [`PinInit::__init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +#[cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait PinInit: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + + /// Initializes `slot`. + /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + /// - `slot` will not move until it is dropped, i.e. it will be pinned. + /// If `Self: Init`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![feature(allocator_api)] + /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; + /// # use pin_init::*; + /// let mtx_init = CMutex::new(42); + /// // Make the initializer print the value. + /// let mtx_init = mtx_init.pin_chain(|mtx| { + /// println!("{:?}", mtx.get_data_mut()); + /// Ok(()) + /// }); + /// ``` + #[inline] + fn pin_chain(self, f: F) -> ChainPinInit + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, __internal::PhantomInvariant::new()) + } +} + +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init(slot: *mut T, init: impl PinInit) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl PinInit for ChainPinInit +where + I: PinInit, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) + } +} + +/// An initializer for `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`] or even the stack (see [`stack_pin_init!`]). Because +/// [`PinInit`] is a super trait, you can use every function that takes it as well. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. +/// +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. +/// +#[cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait Init: PinInit { + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// use pin_init::{init, init_zeroed, Init}; + /// + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// println!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- init_zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + #[inline] + fn chain(self, f: F) -> ChainInit + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, __internal::PhantomInvariant::new()) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function does not rely on the pinning requirement. +unsafe impl Init for ChainInit +where + I: Init, + F: FnOnce(&mut T) -> Result<(), E>, +{ +} + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl PinInit for ChainInit +where + I: Init, + F: FnOnce(&mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) + } +} + +/// Implement `PinInit` and `Init` for closures. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__init` functions. +struct InitClosure(F, __internal::PhantomInvariant); + +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl Init for InitClosure where + F: FnOnce(*mut T) -> Result<(), E> +{ +} + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl PinInit for InitClosure +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + +/// Creates a new [`PinInit`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - may assume that the `slot` does not move if `T: !Unpin`, +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn pin_init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl PinInit { + InitClosure(f, __internal::PhantomInvariant::new()) +} + +/// Creates a new [`Init`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - the `slot` may move after initialization. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl Init { + InitClosure(f, __internal::PhantomInvariant::new()) +} + +/// Changes the to be initialized type. +/// +/// # Safety +/// +/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a +/// pointer must result in a valid `U`. +#[inline] +pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { + // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety + // requirements. + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) } +} + +/// Changes the to be initialized type. +/// +/// # Safety +/// +/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a +/// pointer must result in a valid `U`. +#[inline] +pub const unsafe fn cast_init(init: impl Init) -> impl Init { + // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety + // requirements. + unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) } +} + +/// An initializer that leaves the memory uninitialized. +/// +/// The initializer is a no-op. The `slot` memory is not changed. +#[inline] +pub fn uninit() -> impl Init, E> { + // SAFETY: The memory is allowed to be uninitialized. + unsafe { init_from_closure(|_| Ok(())) } +} + +/// Array initializer from element initializer. +struct ArrayInit(F, __internal::PhantomInvariant); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl PinInit<[T; N], E> for ArrayInit +where + F: FnMut(usize) -> I, + I: PinInit, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl Drop for ArrayInitGuard { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl Init<[T; N], E> for ArrayInit +where + F: FnMut(usize) -> I, + I: Init, +{ +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # use pin_init::*; +/// use pin_init::init_array_from_fn; +/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +#[inline] +pub fn init_array_from_fn( + make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init, +{ + ArrayInit(make_init, __internal::PhantomInvariant::new()) +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// use pin_init::pin_init_array_from_fn; +/// use std::sync::Arc; +/// let array: Pin; 1_000]>> = +/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +#[inline] +pub fn pin_init_array_from_fn( + make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit, +{ + ArrayInit(make_init, __internal::PhantomInvariant::new()) +} + +/// Construct an initializer in a closure and run it. +/// +/// Returns an initializer that first runs the closure and then the initializer returned by it. +/// +/// See also [`init_scope`]. +/// +/// # Examples +/// +/// ``` +/// # use pin_init::*; +/// # #[pin_data] +/// # struct Foo { a: u64, b: isize } +/// # struct Bar { a: u32, b: isize } +/// # fn lookup_bar() -> Result { todo!() } +/// # struct Error; +/// fn init_foo() -> impl PinInit { +/// pin_init_scope(|| { +/// let bar = lookup_bar()?; +/// Ok(pin_init!(Foo { a: bar.a.into(), b: bar.b }? Error)) +/// }) +/// } +/// ``` +/// +/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the +/// initializer itself will fail with that error. If it returned `Ok`, then it will run the +/// initializer returned by the [`pin_init!`] invocation. +#[inline] +pub fn pin_init_scope(make_init: F) -> impl PinInit +where + F: FnOnce() -> Result, + I: PinInit, +{ + // SAFETY: + // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. + unsafe { + pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { + let init = make_init()?; + init.__init(slot) + }) + } +} + +/// Construct an initializer in a closure and run it. +/// +/// Returns an initializer that first runs the closure and then the initializer returned by it. +/// +/// See also [`pin_init_scope`]. +/// +/// # Examples +/// +/// ``` +/// # use pin_init::*; +/// # struct Foo { a: u64, b: isize } +/// # struct Bar { a: u32, b: isize } +/// # fn lookup_bar() -> Result { todo!() } +/// # struct Error; +/// fn init_foo() -> impl Init { +/// init_scope(|| { +/// let bar = lookup_bar()?; +/// Ok(init!(Foo { a: bar.a.into(), b: bar.b }? Error)) +/// }) +/// } +/// ``` +/// +/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the +/// initializer itself will fail with that error. If it returned `Ok`, then it will run the +/// initializer returned by the [`init!`] invocation. +#[inline] +pub fn init_scope(make_init: F) -> impl Init +where + F: FnOnce() -> Result, + I: Init, +{ + // SAFETY: + // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. + unsafe { + init_from_closure(move |slot: *mut T| -> Result<(), E> { + let init = make_init()?; + init.__init(slot) + }) + } +} + +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl Init for T {} + +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of +// `slot`. Additionally, all pinning invariants of `T` are upheld. +unsafe impl PinInit for T { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self) }; + Ok(()) + } +} + +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl Init for Result {} + +// SAFETY: when the `__init` function returns with +// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. +// - `Err(err)`, slot was not written to. +unsafe impl PinInit for Result { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self?) }; + Ok(()) + } +} + +/// Smart pointer containing uninitialized memory and that can write a value. +pub trait InPlaceWrite { + /// The type `Self` turns into when the contents are initialized. + type Initialized; + + /// Use the given initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_init(self, init: impl Init) -> Result; + + /// Use the given pin-initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_pin_init(self, init: impl PinInit) -> Result, E>; +} + +impl InPlaceWrite for &'static mut MaybeUninit { + type Initialized = &'static mut T; + + #[inline] + fn write_init(self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + + // SAFETY: `slot` is a valid pointer to uninitialized memory. + unsafe { init.__init(slot)? }; + + // SAFETY: The above call initialized the memory. + unsafe { Ok(self.assume_init_mut()) } + } + + #[inline] + fn write_pin_init(self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + + // SAFETY: `slot` is a valid pointer to uninitialized memory. + // + // The `'static` borrow guarantees the data will not be + // moved/invalidated until it gets dropped (which is never). + unsafe { init.__init(slot)? }; + + // SAFETY: The above call initialized the memory. + Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) + } +} + +/// Trait facilitating pinned destruction. +/// +/// Use [`pinned_drop`] to implement this trait safely: +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// use core::pin::Pin; +/// #[pin_data(PinnedDrop)] +/// struct Foo { +/// #[pin] +/// mtx: CMutex, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for Foo { +/// fn drop(self: Pin<&mut Self>) { +/// println!("Foo is being dropped!"); +/// } +/// } +/// ``` +/// +/// # Safety +/// +/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. +pub unsafe trait PinnedDrop: __internal::HasPinData { + /// Executes the pinned destructor of this type. + /// + /// While this function is marked safe, it is actually unsafe to call it manually. For this + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code + /// and thus prevents this function from being called where it should not. + /// + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute + /// automatically. + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); +} + +/// Marker trait for types that can be initialized by writing just zeroes. +/// +/// # Safety +/// +/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, +/// this is not UB: +/// +/// ```rust,ignore +/// let val: Self = unsafe { core::mem::zeroed() }; +/// ``` +pub unsafe trait Zeroable { + /// Create a new zeroed `Self`. + /// + /// The returned initializer will write `0x00` to every byte of the given `slot`. + #[inline] + fn init_zeroed() -> impl Init + where + Self: Sized, + { + init_zeroed() + } + + /// Create a `Self` consisting of all zeroes. + /// + /// Whenever a type implements [`Zeroable`], this function should be preferred over + /// [`core::mem::zeroed()`] or using `MaybeUninit::zeroed().assume_init()`. + /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// + /// # Examples + /// + /// ``` + /// use pin_init::Zeroable; + /// + /// #[derive(Zeroable)] + /// struct Point { + /// x: u32, + /// y: u32, + /// } + /// + /// let point: Point = Zeroable::zeroed(); + /// assert_eq!(point.x, 0); + /// assert_eq!(point.y, 0); + /// ``` + #[inline] + fn zeroed() -> Self + where + Self: Sized, + { + zeroed() + } +} + +/// Create an initializer for a zeroed `T`. +/// +/// The returned initializer will write `0x00` to every byte of the given `slot`. +#[inline] +pub fn init_zeroed() -> impl Init { + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` + // and because we write all zeroes, the memory is initialized. + unsafe { + init_from_closure(|slot: *mut T| { + slot.write_bytes(0, 1); + Ok(()) + }) + } +} + +/// Create a `T` consisting of all zeroes. +/// +/// Whenever a type implements [`Zeroable`], this function should be preferred over +/// [`core::mem::zeroed()`] or using `MaybeUninit::zeroed().assume_init()`. +/// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// +/// # Examples +/// +/// ``` +/// use pin_init::{Zeroable, zeroed}; +/// +/// #[derive(Zeroable)] +/// struct Point { +/// x: u32, +/// y: u32, +/// } +/// +/// let point: Point = zeroed(); +/// assert_eq!(point.x, 0); +/// assert_eq!(point.y, 0); +/// ``` +#[inline] +pub const fn zeroed() -> T { + // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. + unsafe { core::mem::zeroed() } +} + +macro_rules! impl_zeroable { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? Zeroable for $t {})* + }; +} + +impl_zeroable! { + // SAFETY: All primitives that are allowed to be zero. + bool, + char, + u8, u16, u32, u64, u128, usize, + i8, i16, i32, i64, i128, isize, + f32, f64, + + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // . The Rust Reference + // also has information on undefined behavior: + // . + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {} PhantomData, core::marker::PhantomPinned, (), + + // SAFETY: Type is allowed to take any value, including all zeros. + {} MaybeUninit, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {} UnsafeCell, + + // SAFETY: `null` pointer is valid. + // + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be + // null. + // + // When `Pointee` gets stabilized, we could use + // `T: ?Sized where ::Metadata: Zeroable` + {} *mut T, {} *const T, + + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be + // zero. + {} *mut [T], {} *const [T], *mut str, *const str, + + // SAFETY: `T` is `Zeroable`. + {} [T; N], {} Wrapping, +} + +macro_rules! impl_tuple_zeroable { + ($first:ident, $(,)?) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for tuples up to 10 items long. + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable> Zeroable for ($first,) {} + }; + ($first:ident, $($t:ident),* $(,)?) => { + #[cfg_attr(doc, doc(hidden))] + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} + impl_tuple_zeroable!($($t),* ,); + } +} + +impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); + +/// Marker trait for types that allow `Option` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl Zeroable for Option {} + +macro_rules! impl_fn_zeroable_option { + ([$($abi:literal),* $(,)?] $args:tt) => { + $(impl_fn_zeroable_option!({extern $abi} $args);)* + $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)* + }; + ({$($prefix:tt)*} {$(,)?}) => {}; + ({$($prefix:tt)*} {$ret:ident, $arg:ident $(,)?}) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for function pointers with up to 20 arity. + // SAFETY: function pointers are part of the option layout optimization: + // . + unsafe impl<$ret, $arg> ZeroableOption for $($prefix)* fn($arg) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$arg,}); + }; + ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => { + #[cfg_attr(doc, doc(hidden))] + // SAFETY: function pointers are part of the option layout optimization: + // . + unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$($rest),*,}); + }; +} + +impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U }); + +macro_rules! impl_zeroable_option { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? ZeroableOption for $t {})* + }; +} + +impl_zeroable_option! { + // SAFETY: `Option<&T>` is part of the option layout optimization guarantee: + // . + {} &T, + // SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: + // . + {} &mut T, + // SAFETY: `Option>` is part of the option layout optimization guarantee: + // . + {} NonNull, + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // ). + NonZero, NonZero, NonZero, NonZero, NonZero, NonZero, + NonZero, NonZero, NonZero, NonZero, NonZero, NonZero, +} + +/// This trait allows creating an instance of `Self` which contains exactly one +/// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning). +/// +/// This is useful when using wrapper `struct`s like [`UnsafeCell`] or with new-type `struct`s. +/// +/// # Examples +/// +/// ``` +/// # use core::cell::UnsafeCell; +/// # use pin_init::{pin_data, pin_init, Wrapper}; +/// +/// #[pin_data] +/// struct Foo {} +/// +/// #[pin_data] +/// struct Bar { +/// #[pin] +/// content: UnsafeCell +/// }; +/// +/// let foo_initializer = pin_init!(Foo{}); +/// let initializer = pin_init!(Bar { +/// content <- UnsafeCell::pin_init(foo_initializer) +/// }); +/// ``` +pub trait Wrapper { + /// Creates an pin-initializer for a [`Self`] containing `T` from the `value_init` initializer. + fn pin_init(value_init: impl PinInit) -> impl PinInit; +} + +impl Wrapper for UnsafeCell { + #[inline] + fn pin_init(value_init: impl PinInit) -> impl PinInit { + // SAFETY: `UnsafeCell` has a compatible layout to `T`. + unsafe { cast_pin_init(value_init) } + } +} + +impl Wrapper for MaybeUninit { + #[inline] + fn pin_init(value_init: impl PinInit) -> impl PinInit { + // SAFETY: `MaybeUninit` has a compatible layout to `T`. + unsafe { cast_pin_init(value_init) } + } +} + +#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] +impl Wrapper for core::pin::UnsafePinned { + #[inline] + fn pin_init(init: impl PinInit) -> impl PinInit { + // SAFETY: `UnsafePinned` has a compatible layout to `T`. + unsafe { cast_pin_init(init) } + } +} diff --git a/rust/proc-macro2/README.md b/rust/proc-macro2/README.md new file mode 100644 index 000000000..af044fee4 --- /dev/null +++ b/rust/proc-macro2/README.md @@ -0,0 +1,13 @@ +# `proc-macro2` + +These source files come from the Rust `proc-macro2` crate, version +1.0.101 (released 2025-08-16), hosted in the + repository, licensed under +"Apache-2.0 OR MIT" and only modified to add the SPDX license +identifiers and to remove the `unicode-ident` dependency. + +For copyright details, please see: + + https://github.com/dtolnay/proc-macro2/blob/1.0.101/README.md#license + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-APACHE + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-MIT diff --git a/rust/proc-macro2/detection.rs b/rust/proc-macro2/detection.rs new file mode 100644 index 000000000..3de448cb2 --- /dev/null +++ b/rust/proc-macro2/detection.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use core::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Once; + +static WORKS: AtomicUsize = AtomicUsize::new(0); +static INIT: Once = Once::new(); + +pub(crate) fn inside_proc_macro() -> bool { + match WORKS.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + + INIT.call_once(initialize); + inside_proc_macro() +} + +pub(crate) fn force_fallback() { + WORKS.store(1, Ordering::Relaxed); +} + +pub(crate) fn unforce_fallback() { + initialize(); +} + +#[cfg(not(no_is_available))] +fn initialize() { + let available = proc_macro::is_available(); + WORKS.store(available as usize + 1, Ordering::Relaxed); +} + +// Swap in a null panic hook to avoid printing "thread panicked" to stderr, +// then use catch_unwind to determine whether the compiler's proc_macro is +// working. When proc-macro2 is used from outside of a procedural macro all +// of the proc_macro crate's APIs currently panic. +// +// The Once is to prevent the possibility of this ordering: +// +// thread 1 calls take_hook, gets the user's original hook +// thread 1 calls set_hook with the null hook +// thread 2 calls take_hook, thinks null hook is the original hook +// thread 2 calls set_hook with the null hook +// thread 1 calls set_hook with the actual original hook +// thread 2 calls set_hook with what it thinks is the original hook +// +// in which the user's hook has been lost. +// +// There is still a race condition where a panic in a different thread can +// happen during the interval that the user's original panic hook is +// unregistered such that their hook is incorrectly not called. This is +// sufficiently unlikely and less bad than printing panic messages to stderr +// on correct use of this crate. Maybe there is a libstd feature request +// here. For now, if a user needs to guarantee that this failure mode does +// not occur, they need to call e.g. `proc_macro2::Span::call_site()` from +// the main thread before launching any other threads. +#[cfg(no_is_available)] +fn initialize() { + use std::panic::{self, PanicInfo}; + + type PanicHook = dyn Fn(&PanicInfo) + Sync + Send + 'static; + + let null_hook: Box = Box::new(|_panic_info| { /* ignore */ }); + let sanity_check = &*null_hook as *const PanicHook; + let original_hook = panic::take_hook(); + panic::set_hook(null_hook); + + let works = panic::catch_unwind(proc_macro::Span::call_site).is_ok(); + WORKS.store(works as usize + 1, Ordering::Relaxed); + + let hopefully_null_hook = panic::take_hook(); + panic::set_hook(original_hook); + if sanity_check != &*hopefully_null_hook { + panic!("observed race condition in proc_macro2::inside_proc_macro"); + } +} diff --git a/rust/proc-macro2/extra.rs b/rust/proc-macro2/extra.rs new file mode 100644 index 000000000..55feb5ec7 --- /dev/null +++ b/rust/proc-macro2/extra.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Items which do not have a correspondence to any API in the proc_macro crate, +//! but are necessary to include in proc-macro2. + +use crate::fallback; +use crate::imp; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use crate::Span; +use core::fmt::{self, Debug}; + +/// Invalidate any `proc_macro2::Span` that exist on the current thread. +/// +/// The implementation of `Span` uses thread-local data structures and this +/// function clears them. Calling any method on a `Span` on the current thread +/// created prior to the invalidation will return incorrect values or crash. +/// +/// This function is useful for programs that process more than 232 +/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2 +/// uses 32-bit source locations, and these wrap around when the total source +/// code processed by the same thread exceeds 232 bytes (4 +/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can +/// return wrong data. +/// +/// # Example +/// +/// As of late 2023, there is 200 GB of Rust code published on crates.io. +/// Looking at just the newest version of every crate, it is 16 GB of code. So a +/// workload that involves parsing it all would overflow a 32-bit source +/// location unless spans are being invalidated. +/// +/// ``` +/// use flate2::read::GzDecoder; +/// use std::ffi::OsStr; +/// use std::io::{BufReader, Read}; +/// use std::str::FromStr; +/// use tar::Archive; +/// +/// rayon::scope(|s| { +/// for krate in every_version_of_every_crate() { +/// s.spawn(move |_| { +/// proc_macro2::extra::invalidate_current_thread_spans(); +/// +/// let reader = BufReader::new(krate); +/// let tar = GzDecoder::new(reader); +/// let mut archive = Archive::new(tar); +/// for entry in archive.entries().unwrap() { +/// let mut entry = entry.unwrap(); +/// let path = entry.path().unwrap(); +/// if path.extension() != Some(OsStr::new("rs")) { +/// continue; +/// } +/// let mut content = String::new(); +/// entry.read_to_string(&mut content).unwrap(); +/// match proc_macro2::TokenStream::from_str(&content) { +/// Ok(tokens) => {/* ... */}, +/// Err(_) => continue, +/// } +/// } +/// }); +/// } +/// }); +/// # +/// # fn every_version_of_every_crate() -> Vec { +/// # Vec::new() +/// # } +/// ``` +/// +/// # Panics +/// +/// This function is not applicable to and will panic if called from a +/// procedural macro. +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub fn invalidate_current_thread_spans() { + crate::imp::invalidate_current_thread_spans(); +} + +/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together +/// in a more compact representation than holding those 2 spans individually. +/// +/// [`Group`]: crate::Group +#[derive(Copy, Clone)] +pub struct DelimSpan { + inner: DelimSpanEnum, + _marker: ProcMacroAutoTraits, +} + +#[derive(Copy, Clone)] +enum DelimSpanEnum { + #[cfg(wrap_proc_macro)] + Compiler { + join: proc_macro::Span, + open: proc_macro::Span, + close: proc_macro::Span, + }, + Fallback(fallback::Span), +} + +impl DelimSpan { + pub(crate) fn new(group: &imp::Group) -> Self { + #[cfg(wrap_proc_macro)] + let inner = match group { + imp::Group::Compiler(group) => DelimSpanEnum::Compiler { + join: group.span(), + open: group.span_open(), + close: group.span_close(), + }, + imp::Group::Fallback(group) => DelimSpanEnum::Fallback(group.span()), + }; + + #[cfg(not(wrap_proc_macro))] + let inner = DelimSpanEnum::Fallback(group.span()); + + DelimSpan { + inner, + _marker: MARKER, + } + } + + /// Returns a span covering the entire delimited group. + pub fn join(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { join, .. } => Span::_new(imp::Span::Compiler(*join)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(*span), + } + } + + /// Returns a span for the opening punctuation of the group only. + pub fn open(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { open, .. } => Span::_new(imp::Span::Compiler(*open)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.first_byte()), + } + } + + /// Returns a span for the closing punctuation of the group only. + pub fn close(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { close, .. } => Span::_new(imp::Span::Compiler(*close)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.last_byte()), + } + } +} + +impl Debug for DelimSpan { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.join(), f) + } +} diff --git a/rust/proc-macro2/fallback.rs b/rust/proc-macro2/fallback.rs new file mode 100644 index 000000000..9b43c97df --- /dev/null +++ b/rust/proc-macro2/fallback.rs @@ -0,0 +1,1258 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(wrap_proc_macro)] +use crate::imp; +#[cfg(span_locations)] +use crate::location::LineColumn; +use crate::parse::{self, Cursor}; +use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; +use crate::{Delimiter, Spacing, TokenTree}; +#[cfg(all(span_locations, not(fuzzing)))] +use alloc::collections::BTreeMap; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cell::RefCell; +#[cfg(span_locations)] +use core::cmp; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display, Write}; +use core::mem::ManuallyDrop; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::ptr; +use core::str; +#[cfg(feature = "proc-macro")] +use core::str::FromStr; +use std::ffi::CStr; +#[cfg(wrap_proc_macro)] +use std::panic; +#[cfg(span_locations)] +use std::path::PathBuf; + +/// Force use of proc-macro2's fallback implementation of the API for now, even +/// if the compiler's implementation is available. +pub fn force() { + #[cfg(wrap_proc_macro)] + crate::detection::force_fallback(); +} + +/// Resume using the compiler's implementation of the proc macro API if it is +/// available. +pub fn unforce() { + #[cfg(wrap_proc_macro)] + crate::detection::unforce_fallback(); +} + +#[derive(Clone)] +pub(crate) struct TokenStream { + inner: RcVec, +} + +#[derive(Debug)] +pub(crate) struct LexError { + pub(crate) span: Span, +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn call_site() -> Self { + LexError { + span: Span::call_site(), + } + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + TokenStream { + inner: RcVecBuilder::new().build(), + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + // Create a dummy file & add it to the source map + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } + + parse::token_stream(cursor) + } + + #[cfg(feature = "proc-macro")] + pub(crate) fn from_str_unchecked(src: &str) -> Self { + Self::from_str_checked(src).unwrap() + } + + pub(crate) fn is_empty(&self) -> bool { + self.inner.len() == 0 + } + + fn take_inner(self) -> RcVecBuilder { + let nodrop = ManuallyDrop::new(self); + unsafe { ptr::read(&nodrop.inner) }.make_owned() + } +} + +fn push_token_from_proc_macro(mut vec: RcVecMut, token: TokenTree) { + // https://github.com/dtolnay/proc-macro2/issues/235 + match token { + TokenTree::Literal(crate::Literal { + #[cfg(wrap_proc_macro)] + inner: crate::imp::Literal::Fallback(literal), + #[cfg(not(wrap_proc_macro))] + inner: literal, + .. + }) if literal.repr.starts_with('-') => { + push_negative_literal(vec, literal); + } + _ => vec.push(token), + } + + #[cold] + fn push_negative_literal(mut vec: RcVecMut, mut literal: Literal) { + literal.repr.remove(0); + let mut punct = crate::Punct::new('-', Spacing::Alone); + punct.set_span(crate::Span::_new_fallback(literal.span)); + vec.push(TokenTree::Punct(punct)); + vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); + } +} + +// Nonrecursive to prevent stack overflow. +impl Drop for TokenStream { + fn drop(&mut self) { + let mut stack = Vec::new(); + let mut current = match self.inner.get_mut() { + Some(inner) => inner.take().into_iter(), + None => return, + }; + loop { + while let Some(token) = current.next() { + let group = match token { + TokenTree::Group(group) => group.inner, + _ => continue, + }; + #[cfg(wrap_proc_macro)] + let group = match group { + crate::imp::Group::Fallback(group) => group, + crate::imp::Group::Compiler(_) => continue, + }; + let mut group = group; + if let Some(inner) = group.stream.inner.get_mut() { + stack.push(current); + current = inner.take().into_iter(); + } + } + match stack.pop() { + Some(next) => current = next, + None => return, + } + } + } +} + +pub(crate) struct TokenStreamBuilder { + inner: RcVecBuilder, +} + +impl TokenStreamBuilder { + pub(crate) fn new() -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::new(), + } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::with_capacity(cap), + } + } + + pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) { + self.inner.push(tt); + } + + pub(crate) fn build(self) -> TokenStream { + TokenStream { + inner: self.inner.build(), + } + } +} + +#[cfg(span_locations)] +fn get_cursor(src: &str) -> Cursor { + #[cfg(fuzzing)] + return Cursor { rest: src, off: 1 }; + + // Create a dummy file & add it to the source map + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let mut sm = sm.borrow_mut(); + let span = sm.add_file(src); + Cursor { + rest: src, + off: span.lo, + } + }) +} + +#[cfg(not(span_locations))] +fn get_cursor(src: &str) -> Cursor { + Cursor { rest: src } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("cannot parse string into token stream") + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut joint = false; + for (i, tt) in self.inner.iter().enumerate() { + if i != 0 && !joint { + write!(f, " ")?; + } + joint = false; + match tt { + TokenTree::Group(tt) => Display::fmt(tt, f), + TokenTree::Ident(tt) => Display::fmt(tt, f), + TokenTree::Punct(tt) => { + joint = tt.spacing() == Spacing::Joint; + Display::fmt(tt, f) + } + TokenTree::Literal(tt) => Display::fmt(tt, f), + }?; + } + + Ok(()) + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(feature = "proc-macro")] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +#[cfg(feature = "proc-macro")] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +impl From for TokenStream { + fn from(tree: TokenTree) -> Self { + let mut stream = RcVecBuilder::new(); + push_token_from_proc_macro(stream.as_mut(), tree); + TokenStream { + inner: stream.build(), + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(tokens: I) -> Self { + let mut stream = TokenStream::new(); + stream.extend(tokens); + stream + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut v = RcVecBuilder::new(); + + for stream in streams { + v.extend(stream.take_inner()); + } + + TokenStream { inner: v.build() } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, tokens: I) { + let mut vec = self.inner.make_mut(); + tokens + .into_iter() + .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token)); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.make_mut().extend(streams.into_iter().flatten()); + } +} + +pub(crate) type TokenTreeIter = RcVecIntoIter; + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + self.take_inner().into_iter() + } +} + +#[cfg(all(span_locations, not(fuzzing)))] +thread_local! { + static SOURCE_MAP: RefCell = RefCell::new(SourceMap { + // Start with a single dummy file which all call_site() and def_site() + // spans reference. + files: vec![FileInfo { + source_text: String::new(), + span: Span { lo: 0, hi: 0 }, + lines: vec![0], + char_index_to_byte_offset: BTreeMap::new(), + }], + }); +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1)); +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct FileInfo { + source_text: String, + span: Span, + lines: Vec, + char_index_to_byte_offset: BTreeMap, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl FileInfo { + fn offset_line_column(&self, offset: usize) -> LineColumn { + assert!(self.span_within(Span { + lo: offset as u32, + hi: offset as u32, + })); + let offset = offset - self.span.lo as usize; + match self.lines.binary_search(&offset) { + Ok(found) => LineColumn { + line: found + 1, + column: 0, + }, + Err(idx) => LineColumn { + line: idx, + column: offset - self.lines[idx - 1], + }, + } + } + + fn span_within(&self, span: Span) -> bool { + span.lo >= self.span.lo && span.hi <= self.span.hi + } + + fn byte_range(&mut self, span: Span) -> Range { + let lo_char = (span.lo - self.span.lo) as usize; + + // Look up offset of the largest already-computed char index that is + // less than or equal to the current requested one. We resume counting + // chars from that point. + let (&last_char_index, &last_byte_offset) = self + .char_index_to_byte_offset + .range(..=lo_char) + .next_back() + .unwrap_or((&0, &0)); + + let lo_byte = if last_char_index == lo_char { + last_byte_offset + } else { + let total_byte_offset = match self.source_text[last_byte_offset..] + .char_indices() + .nth(lo_char - last_char_index) + { + Some((additional_offset, _ch)) => last_byte_offset + additional_offset, + None => self.source_text.len(), + }; + self.char_index_to_byte_offset + .insert(lo_char, total_byte_offset); + total_byte_offset + }; + + let trunc_lo = &self.source_text[lo_byte..]; + let char_len = (span.hi - span.lo) as usize; + lo_byte..match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => lo_byte + offset, + None => self.source_text.len(), + } + } + + fn source_text(&mut self, span: Span) -> String { + let byte_range = self.byte_range(span); + self.source_text[byte_range].to_owned() + } +} + +/// Computes the offsets of each line in the given source string +/// and the total number of characters +#[cfg(all(span_locations, not(fuzzing)))] +fn lines_offsets(s: &str) -> (usize, Vec) { + let mut lines = vec![0]; + let mut total = 0; + + for ch in s.chars() { + total += 1; + if ch == '\n' { + lines.push(total); + } + } + + (total, lines) +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct SourceMap { + files: Vec, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl SourceMap { + fn next_start_pos(&self) -> u32 { + // Add 1 so there's always space between files. + // + // We'll always have at least 1 file, as we initialize our files list + // with a dummy file. + self.files.last().unwrap().span.hi + 1 + } + + fn add_file(&mut self, src: &str) -> Span { + let (len, lines) = lines_offsets(src); + let lo = self.next_start_pos(); + let span = Span { + lo, + hi: lo + (len as u32), + }; + + self.files.push(FileInfo { + source_text: src.to_owned(), + span, + lines, + // Populated lazily by source_text(). + char_index_to_byte_offset: BTreeMap::new(), + }); + + span + } + + fn find(&self, span: Span) -> usize { + match self.files.binary_search_by(|file| { + if file.span.hi < span.lo { + Ordering::Less + } else if file.span.lo > span.hi { + Ordering::Greater + } else { + assert!(file.span_within(span)); + Ordering::Equal + } + }) { + Ok(i) => i, + Err(_) => unreachable!("Invalid span with no related FileInfo!"), + } + } + + fn filepath(&self, span: Span) -> String { + let i = self.find(span); + if i == 0 { + "".to_owned() + } else { + format!("", i) + } + } + + fn fileinfo(&self, span: Span) -> &FileInfo { + let i = self.find(span); + &self.files[i] + } + + fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo { + let i = self.find(span); + &mut self.files[i] + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Span { + #[cfg(span_locations)] + pub(crate) lo: u32, + #[cfg(span_locations)] + pub(crate) hi: u32, +} + +impl Span { + #[cfg(not(span_locations))] + pub(crate) fn call_site() -> Self { + Span {} + } + + #[cfg(span_locations)] + pub(crate) fn call_site() -> Self { + Span { lo: 0, hi: 0 } + } + + pub(crate) fn mixed_site() -> Self { + Span::call_site() + } + + #[cfg(procmacro2_semver_exempt)] + pub(crate) fn def_site() -> Self { + Span::call_site() + } + + pub(crate) fn resolved_at(&self, _other: Span) -> Span { + // Stable spans consist only of line/column information, so + // `resolved_at` and `located_at` only select which span the + // caller wants line/column information from. + *self + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + other + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + #[cfg(fuzzing)] + return 0..0; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + 0..0 + } else { + SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self)) + } + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.lo as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.hi as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + #[cfg(fuzzing)] + return "".to_owned(); + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + sm.filepath(*self) + }) + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + None + } + + #[cfg(not(span_locations))] + pub(crate) fn join(&self, _other: Span) -> Option { + Some(Span {}) + } + + #[cfg(span_locations)] + pub(crate) fn join(&self, other: Span) -> Option { + #[cfg(fuzzing)] + return { + let _ = other; + None + }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + // If `other` is not within the same FileInfo as us, return None. + if !sm.fileinfo(*self).span_within(other) { + return None; + } + Some(Span { + lo: cmp::min(self.lo, other.lo), + hi: cmp::max(self.hi, other.hi), + }) + }) + } + + #[cfg(not(span_locations))] + pub(crate) fn source_text(&self) -> Option { + None + } + + #[cfg(span_locations)] + pub(crate) fn source_text(&self) -> Option { + #[cfg(fuzzing)] + return None; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + None + } else { + Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + } + } + } + + #[cfg(not(span_locations))] + pub(crate) fn first_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn first_byte(self) -> Self { + Span { + lo: self.lo, + hi: cmp::min(self.lo.saturating_add(1), self.hi), + } + } + + #[cfg(not(span_locations))] + pub(crate) fn last_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn last_byte(self) -> Self { + Span { + lo: cmp::max(self.hi.saturating_sub(1), self.lo), + hi: self.hi, + } + } + + #[cfg(span_locations)] + fn is_call_site(&self) -> bool { + self.lo == 0 && self.hi == 0 + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + #[cfg(span_locations)] + return write!(f, "bytes({}..{})", self.lo, self.hi); + + #[cfg(not(span_locations))] + write!(f, "Span") + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + #[cfg(span_locations)] + { + if span.is_call_site() { + return; + } + } + + if cfg!(span_locations) { + debug.field("span", &span); + } +} + +#[derive(Clone)] +pub(crate) struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + delimiter, + stream, + span: Span::call_site(), + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + self.delimiter + } + + pub(crate) fn stream(&self) -> TokenStream { + self.stream.clone() + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn span_open(&self) -> Span { + self.span.first_byte() + } + + pub(crate) fn span_close(&self) -> Span { + self.span.last_byte() + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl Display for Group { + // We attempt to match libproc_macro's formatting. + // Empty parens: () + // Nonempty parens: (...) + // Empty brackets: [] + // Nonempty brackets: [...] + // Empty braces: { } + // Nonempty braces: { ... } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (open, close) = match self.delimiter { + Delimiter::Parenthesis => ("(", ")"), + Delimiter::Brace => ("{ ", "}"), + Delimiter::Bracket => ("[", "]"), + Delimiter::None => ("", ""), + }; + + f.write_str(open)?; + Display::fmt(&self.stream, f)?; + if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() { + f.write_str(" ")?; + } + f.write_str(close)?; + + Ok(()) + } +} + +impl Debug for Group { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Group"); + debug.field("delimiter", &self.delimiter); + debug.field("stream", &self.stream); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Ident { + sym: Box, + span: Span, + raw: bool, +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + validate_ident(string); + Ident::new_unchecked(string, span) + } + + pub(crate) fn new_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: false, + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + validate_ident_raw(string); + Ident::new_raw_unchecked(string, span) + } + + pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: true, + } + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +pub(crate) fn is_ident_start(c: char) -> bool { + c == '_' || c.is_ascii_alphabetic() +} + +pub(crate) fn is_ident_continue(c: char) -> bool { + c == '_' || c.is_ascii_alphanumeric() +} + +#[track_caller] +fn validate_ident(string: &str) { + if string.is_empty() { + panic!("Ident is not allowed to be empty; use Option"); + } + + if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') { + panic!("Ident cannot be a number; use Literal instead"); + } + + fn ident_ok(string: &str) -> bool { + let mut chars = string.chars(); + let first = chars.next().unwrap(); + if !is_ident_start(first) { + return false; + } + for ch in chars { + if !is_ident_continue(ch) { + return false; + } + } + true + } + + if !ident_ok(string) { + panic!("{:?} is not a valid Ident", string); + } +} + +#[track_caller] +fn validate_ident_raw(string: &str) { + validate_ident(string); + + match string { + "_" | "super" | "self" | "Self" | "crate" => { + panic!("`r#{}` cannot be a raw identifier", string); + } + _ => {} + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.sym == other.sym && self.raw == other.raw + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + if self.raw { + other.starts_with("r#") && *self.sym == other[2..] + } else { + *self.sym == *other + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.raw { + f.write_str("r#")?; + } + Display::fmt(&self.sym, f) + } +} + +#[allow(clippy::missing_fields_in_debug)] +impl Debug for Ident { + // Ident(proc_macro), Ident(r#union) + #[cfg(not(span_locations))] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_tuple("Ident"); + debug.field(&format_args!("{}", self)); + debug.finish() + } + + // Ident { + // sym: proc_macro, + // span: bytes(128..138) + // } + #[cfg(span_locations)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", self)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Literal { + pub(crate) repr: String, + span: Span, +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(format!(concat!("{}", stringify!($kind)), n)) + } + )*) +} + +macro_rules! unsuffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(n.to_string()) + } + )*) +} + +impl Literal { + pub(crate) fn _new(repr: String) -> Self { + Literal { + repr, + span: Span::call_site(), + } + } + + pub(crate) fn from_str_checked(repr: &str) -> Result { + let mut cursor = get_cursor(repr); + #[cfg(span_locations)] + let lo = cursor.off; + + let negative = cursor.starts_with_char('-'); + if negative { + cursor = cursor.advance(1); + if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { + return Err(LexError::call_site()); + } + } + + if let Ok((rest, mut literal)) = parse::literal(cursor) { + if rest.is_empty() { + if negative { + literal.repr.insert(0, '-'); + } + literal.span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + return Ok(literal); + } + } + Err(LexError::call_site()) + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(repr.to_owned()) + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_numbers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn string(string: &str) -> Literal { + let mut repr = String::with_capacity(string.len() + 2); + repr.push('"'); + escape_utf8(string, &mut repr); + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn character(ch: char) -> Literal { + let mut repr = String::new(); + repr.push('\''); + if ch == '"' { + // escape_debug turns this into '\"' which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + let mut repr = "b'".to_string(); + #[allow(clippy::match_overlapping_arm)] + match byte { + b'\0' => repr.push_str(r"\0"), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'\'' => repr.push_str(r"\'"), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(byte as char), + _ => { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + let mut repr = "b\"".to_string(); + let mut bytes = bytes.iter(); + while let Some(&b) = bytes.next() { + #[allow(clippy::match_overlapping_arm)] + match b { + b'\0' => repr.push_str(match bytes.as_slice().first() { + // circumvent clippy::octal_escapes lint + Some(b'0'..=b'7') => r"\x00", + _ => r"\0", + }), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'"' => repr.push_str("\\\""), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(b as char), + _ => { + let _ = write!(repr, r"\x{:02X}", b); + } + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + let mut repr = "c\"".to_string(); + let mut bytes = string.to_bytes(); + while !bytes.is_empty() { + let (valid, invalid) = match str::from_utf8(bytes) { + Ok(all_valid) => { + bytes = b""; + (all_valid, bytes) + } + Err(utf8_error) => { + let (valid, rest) = bytes.split_at(utf8_error.valid_up_to()); + let valid = str::from_utf8(valid).unwrap(); + let invalid = utf8_error + .error_len() + .map_or(rest, |error_len| &rest[..error_len]); + bytes = &bytes[valid.len() + invalid.len()..]; + (valid, invalid) + } + }; + escape_utf8(valid, &mut repr); + for &byte in invalid { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + #[cfg(not(span_locations))] + { + let _ = range; + None + } + + #[cfg(span_locations)] + { + use core::ops::Bound; + + let lo = match range.start_bound() { + Bound::Included(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)? + } + Bound::Excluded(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)?.checked_add(1)? + } + Bound::Unbounded => self.span.lo, + }; + let hi = match range.end_bound() { + Bound::Included(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)?.checked_add(1)? + } + Bound::Excluded(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)? + } + Bound::Unbounded => self.span.hi, + }; + if lo <= hi && hi <= self.span.hi { + Some(Span { lo, hi }) + } else { + None + } + } + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.repr, f) + } +} + +impl Debug for Literal { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Literal"); + debug.field("lit", &format_args!("{}", self.repr)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +fn escape_utf8(string: &str, repr: &mut String) { + let mut chars = string.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + r"\x00" + } else { + r"\0" + }, + ); + } else if ch == '\'' { + // escape_debug turns this into "\'" which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + } +} + +#[cfg(feature = "proc-macro")] +pub(crate) trait FromStr2: FromStr { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool; + + #[cfg(wrap_proc_macro)] + fn from_str_checked(src: &str) -> Result { + // Validate using fallback parser, because rustc is incapable of + // returning a recoverable Err for certain invalid token streams, and + // will instead permanently poison the compilation. + if !Self::valid(src) { + return Err(imp::LexError::CompilerPanic); + } + + // Catch panic to work around https://github.com/rust-lang/rust/issues/58736. + match panic::catch_unwind(|| Self::from_str(src)) { + Ok(Ok(ok)) => Ok(ok), + Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)), + Err(_panic) => Err(imp::LexError::CompilerPanic), + } + } + + fn from_str_unchecked(src: &str) -> Self { + Self::from_str(src).unwrap() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::TokenStream { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + TokenStream::from_str_checked(src).is_ok() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::Literal { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + Literal::from_str_checked(src).is_ok() + } +} diff --git a/rust/proc-macro2/lib.rs b/rust/proc-macro2/lib.rs new file mode 100644 index 000000000..5d408943f --- /dev/null +++ b/rust/proc-macro2/lib.rs @@ -0,0 +1,1355 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! [![github]](https://github.com/dtolnay/proc-macro2) [![crates-io]](https://crates.io/crates/proc-macro2) [![docs-rs]](crate) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! A wrapper around the procedural macro API of the compiler's [`proc_macro`] +//! crate. This library serves two purposes: +//! +//! - **Bring proc-macro-like functionality to other contexts like build.rs and +//! main.rs.** Types from `proc_macro` are entirely specific to procedural +//! macros and cannot ever exist in code outside of a procedural macro. +//! Meanwhile `proc_macro2` types may exist anywhere including non-macro code. +//! By developing foundational libraries like [syn] and [quote] against +//! `proc_macro2` rather than `proc_macro`, the procedural macro ecosystem +//! becomes easily applicable to many other use cases and we avoid +//! reimplementing non-macro equivalents of those libraries. +//! +//! - **Make procedural macros unit testable.** As a consequence of being +//! specific to procedural macros, nothing that uses `proc_macro` can be +//! executed from a unit test. In order for helper libraries or components of +//! a macro to be testable in isolation, they must be implemented using +//! `proc_macro2`. +//! +//! [syn]: https://github.com/dtolnay/syn +//! [quote]: https://github.com/dtolnay/quote +//! +//! # Usage +//! +//! The skeleton of a typical procedural macro typically looks like this: +//! +//! ``` +//! extern crate proc_macro; +//! +//! # const IGNORE: &str = stringify! { +//! #[proc_macro_derive(MyDerive)] +//! # }; +//! # #[cfg(wrap_proc_macro)] +//! pub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +//! let input = proc_macro2::TokenStream::from(input); +//! +//! let output: proc_macro2::TokenStream = { +//! /* transform input */ +//! # input +//! }; +//! +//! proc_macro::TokenStream::from(output) +//! } +//! ``` +//! +//! If parsing with [Syn], you'll use [`parse_macro_input!`] instead to +//! propagate parse errors correctly back to the compiler when parsing fails. +//! +//! [`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html +//! +//! # Unstable features +//! +//! The default feature set of proc-macro2 tracks the most recent stable +//! compiler API. Functionality in `proc_macro` that is not yet stable is not +//! exposed by proc-macro2 by default. +//! +//! To opt into the additional APIs available in the most recent nightly +//! compiler, the `procmacro2_semver_exempt` config flag must be passed to +//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As +//! these are unstable APIs that track the nightly compiler, minor versions of +//! proc-macro2 may make breaking changes to them at any time. +//! +//! ```sh +//! RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build +//! ``` +//! +//! Note that this must not only be done for your crate, but for any crate that +//! depends on your crate. This infectious nature is intentional, as it serves +//! as a reminder that you are outside of the normal semver guarantees. +//! +//! Semver exempt methods are marked as such in the proc-macro2 documentation. +//! +//! # Thread-Safety +//! +//! Most types in this crate are `!Sync` because the underlying compiler +//! types make use of thread-local memory, meaning they cannot be accessed from +//! a different thread. + +// Proc-macro2 types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.101")] +#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] +#![cfg_attr(super_unstable, feature(proc_macro_def_site))] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(unsafe_op_in_unsafe_fn)] +#![allow( + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::checked_conversions, + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::incompatible_msrv, + clippy::items_after_statements, + clippy::iter_without_into_iter, + clippy::let_underscore_untyped, + clippy::manual_assert, + clippy::manual_range_contains, + clippy::missing_panics_doc, + clippy::missing_safety_doc, + clippy::must_use_candidate, + clippy::needless_doctest_main, + clippy::needless_lifetimes, + clippy::new_without_default, + clippy::return_self_not_must_use, + clippy::shadow_unrelated, + clippy::trivially_copy_pass_by_ref, + clippy::unnecessary_wraps, + clippy::unused_self, + clippy::used_underscore_binding, + clippy::vec_init_then_push +)] +#![allow(unknown_lints, mismatched_lifetime_syntaxes)] + +#[cfg(all(procmacro2_semver_exempt, wrap_proc_macro, not(super_unstable)))] +compile_error! {"\ + Something is not right. If you've tried to turn on \ + procmacro2_semver_exempt, you need to ensure that it \ + is turned on for the compilation of the proc-macro2 \ + build script as well. +"} + +#[cfg(all( + procmacro2_nightly_testing, + feature = "proc-macro", + not(proc_macro_span) +))] +compile_error! {"\ + Build script probe failed to compile. +"} + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod marker; +mod parse; +mod probe; +mod rcvec; + +#[cfg(wrap_proc_macro)] +mod detection; + +// Public for proc_macro2::fallback::force() and unforce(), but those are quite +// a niche use case so we omit it from rustdoc. +#[doc(hidden)] +pub mod fallback; + +pub mod extra; + +#[cfg(not(wrap_proc_macro))] +use crate::fallback as imp; +#[path = "wrapper.rs"] +#[cfg(wrap_proc_macro)] +mod imp; + +#[cfg(span_locations)] +mod location; + +use crate::extra::DelimSpan; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display}; +use core::hash::{Hash, Hasher}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::str::FromStr; +use std::error::Error; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub use crate::location::LineColumn; + +/// An abstract stream of tokens, or more concretely a sequence of token trees. +/// +/// This type provides interfaces for iterating over token trees and for +/// collecting token trees into one stream. +/// +/// Token stream is both the input and output of `#[proc_macro]`, +/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions. +#[derive(Clone)] +pub struct TokenStream { + inner: imp::TokenStream, + _marker: ProcMacroAutoTraits, +} + +/// Error returned from `TokenStream::from_str`. +pub struct LexError { + inner: imp::LexError, + _marker: ProcMacroAutoTraits, +} + +impl TokenStream { + fn _new(inner: imp::TokenStream) -> Self { + TokenStream { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::TokenStream) -> Self { + TokenStream { + inner: imp::TokenStream::from(inner), + _marker: MARKER, + } + } + + /// Returns an empty `TokenStream` containing no token trees. + pub fn new() -> Self { + TokenStream::_new(imp::TokenStream::new()) + } + + /// Checks if this `TokenStream` is empty. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + +/// `TokenStream::default()` returns an empty stream, +/// i.e. this is equivalent with `TokenStream::new()`. +impl Default for TokenStream { + fn default() -> Self { + TokenStream::new() + } +} + +/// Attempts to break the string into tokens and parse those tokens into a token +/// stream. +/// +/// May fail for a number of reasons, for example, if the string contains +/// unbalanced delimiters or characters not existing in the language. +/// +/// NOTE: Some errors may cause panics instead of returning `LexError`. We +/// reserve the right to change these errors into `LexError`s later. +impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + match imp::TokenStream::from_str_checked(src) { + Ok(tokens) => Ok(TokenStream::_new(tokens)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::_new(imp::TokenStream::from(inner)) + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from(inner.inner) + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + TokenStream::_new(imp::TokenStream::from(token)) + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.extend(streams); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner + .extend(streams.into_iter().map(|stream| stream.inner)); + } +} + +/// Collects a number of token trees into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().collect()) + } +} +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().map(|i| i.inner).collect()) + } +} + +/// Prints the token stream as a string that is supposed to be losslessly +/// convertible back into the same token stream (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Prints token in a form convenient for debugging. +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl LexError { + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Error for LexError {} + +/// A region of source code, along with macro expansion information. +#[derive(Copy, Clone)] +pub struct Span { + inner: imp::Span, + _marker: ProcMacroAutoTraits, +} + +impl Span { + fn _new(inner: imp::Span) -> Self { + Span { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Span) -> Self { + Span { + inner: imp::Span::from(inner), + _marker: MARKER, + } + } + + /// The span of the invocation of the current procedural macro. + /// + /// Identifiers created with this span will be resolved as if they were + /// written directly at the macro call location (call-site hygiene) and + /// other code at the macro call site will be able to refer to them as well. + pub fn call_site() -> Self { + Span::_new(imp::Span::call_site()) + } + + /// The span located at the invocation of the procedural macro, but with + /// local variables, labels, and `$crate` resolved at the definition site + /// of the macro. This is the same hygiene behavior as `macro_rules`. + pub fn mixed_site() -> Self { + Span::_new(imp::Span::mixed_site()) + } + + /// A span that resolves at the macro definition site. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn def_site() -> Self { + Span::_new(imp::Span::def_site()) + } + + /// Creates a new span with the same line/column information as `self` but + /// that resolves symbols as though it were at `other`. + pub fn resolved_at(&self, other: Span) -> Span { + Span::_new(self.inner.resolved_at(other.inner)) + } + + /// Creates a new span with the same name resolution behavior as `self` but + /// with the line/column information of `other`. + pub fn located_at(&self, other: Span) -> Span { + Span::_new(self.inner.located_at(other.inner)) + } + + /// Convert `proc_macro2::Span` to `proc_macro::Span`. + /// + /// This method is available when building with a nightly compiler, or when + /// building with rustc 1.29+ *without* semver exempt features. + /// + /// # Panics + /// + /// Panics if called from outside of a procedural macro. Unlike + /// `proc_macro2::Span`, the `proc_macro::Span` type can only exist within + /// the context of a procedural macro invocation. + #[cfg(wrap_proc_macro)] + pub fn unwrap(self) -> proc_macro::Span { + self.inner.unwrap() + } + + // Soft deprecated. Please use Span::unwrap. + #[cfg(wrap_proc_macro)] + #[doc(hidden)] + pub fn unstable(self) -> proc_macro::Span { + self.unwrap() + } + + /// Returns the span's byte position range in the source file. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned range is only + /// accurate if compiled with a nightly toolchain. The stable toolchain does + /// not have this information available. When executing outside of a + /// procedural macro, such as main.rs or build.rs, the byte range is always + /// accurate regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn byte_range(&self) -> Range { + self.inner.byte_range() + } + + /// Get the starting line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn start(&self) -> LineColumn { + self.inner.start() + } + + /// Get the ending line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn end(&self) -> LineColumn { + self.inner.end() + } + + /// The path to the source file in which this span occurs, for display + /// purposes. + /// + /// This might not correspond to a valid file system path. It might be + /// remapped, or might be an artificial path such as `""`. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn file(&self) -> String { + self.inner.file() + } + + /// The path to the source file in which this span occurs on disk. + /// + /// This is the actual path on disk. It is unaffected by path remapping. + /// + /// This path should not be embedded in the output of the macro; prefer + /// `file()` instead. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn local_file(&self) -> Option { + self.inner.local_file() + } + + /// Create a new span encompassing `self` and `other`. + /// + /// Returns `None` if `self` and `other` are from different files. + /// + /// Warning: the underlying [`proc_macro::Span::join`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn join(&self, other: Span) -> Option { + self.inner.join(other.inner).map(Span::_new) + } + + /// Compares two spans to see if they're equal. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn eq(&self, other: &Span) -> bool { + self.inner.eq(&other.inner) + } + + /// Returns the source text behind a span. This preserves the original + /// source code, including spaces and comments. It only returns a result if + /// the span corresponds to real source code. + /// + /// Note: The observable result of a macro should only rely on the tokens + /// and not on this source text. The result of this function is a best + /// effort to be used for diagnostics only. + pub fn source_text(&self) -> Option { + self.inner.source_text() + } +} + +/// Prints a span in a form convenient for debugging. +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`). +#[derive(Clone)] +pub enum TokenTree { + /// A token stream surrounded by bracket delimiters. + Group(Group), + /// An identifier. + Ident(Ident), + /// A single punctuation character (`+`, `,`, `$`, etc.). + Punct(Punct), + /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc. + Literal(Literal), +} + +impl TokenTree { + /// Returns the span of this tree, delegating to the `span` method of + /// the contained token or a delimited stream. + pub fn span(&self) -> Span { + match self { + TokenTree::Group(t) => t.span(), + TokenTree::Ident(t) => t.span(), + TokenTree::Punct(t) => t.span(), + TokenTree::Literal(t) => t.span(), + } + } + + /// Configures the span for *only this token*. + /// + /// Note that if this token is a `Group` then this method will not configure + /// the span of each of the internal tokens, this will simply delegate to + /// the `set_span` method of each variant. + pub fn set_span(&mut self, span: Span) { + match self { + TokenTree::Group(t) => t.set_span(span), + TokenTree::Ident(t) => t.set_span(span), + TokenTree::Punct(t) => t.set_span(span), + TokenTree::Literal(t) => t.set_span(span), + } + } +} + +impl From for TokenTree { + fn from(g: Group) -> Self { + TokenTree::Group(g) + } +} + +impl From for TokenTree { + fn from(g: Ident) -> Self { + TokenTree::Ident(g) + } +} + +impl From for TokenTree { + fn from(g: Punct) -> Self { + TokenTree::Punct(g) + } +} + +impl From for TokenTree { + fn from(g: Literal) -> Self { + TokenTree::Literal(g) + } +} + +/// Prints the token tree as a string that is supposed to be losslessly +/// convertible back into the same token tree (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenTree::Group(t) => Display::fmt(t, f), + TokenTree::Ident(t) => Display::fmt(t, f), + TokenTree::Punct(t) => Display::fmt(t, f), + TokenTree::Literal(t) => Display::fmt(t, f), + } + } +} + +/// Prints token tree in a form convenient for debugging. +impl Debug for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Each of these has the name in the struct type in the derived debug, + // so don't bother with an extra layer of indirection + match self { + TokenTree::Group(t) => Debug::fmt(t, f), + TokenTree::Ident(t) => { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", t)); + imp::debug_span_field_if_nontrivial(&mut debug, t.span().inner); + debug.finish() + } + TokenTree::Punct(t) => Debug::fmt(t, f), + TokenTree::Literal(t) => Debug::fmt(t, f), + } + } +} + +/// A delimited token stream. +/// +/// A `Group` internally contains a `TokenStream` which is surrounded by +/// `Delimiter`s. +#[derive(Clone)] +pub struct Group { + inner: imp::Group, +} + +/// Describes how a sequence of token trees is delimited. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Delimiter { + /// `( ... )` + Parenthesis, + /// `{ ... }` + Brace, + /// `[ ... ]` + Bracket, + /// `∅ ... ∅` + /// + /// An invisible delimiter, that may, for example, appear around tokens + /// coming from a "macro variable" `$var`. It is important to preserve + /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`. + /// Invisible delimiters may not survive roundtrip of a token stream through + /// a string. + /// + ///

+ /// + /// Note: rustc currently can ignore the grouping of tokens delimited by `None` in the output + /// of a proc_macro. Only `None`-delimited groups created by a macro_rules macro in the input + /// of a proc_macro macro are preserved, and only in very specific circumstances. + /// Any `None`-delimited groups (re)created by a proc_macro will therefore not preserve + /// operator priorities as indicated above. The other `Delimiter` variants should be used + /// instead in this context. This is a rustc bug. For details, see + /// [rust-lang/rust#67062](https://github.com/rust-lang/rust/issues/67062). + /// + ///
+ None, +} + +impl Group { + fn _new(inner: imp::Group) -> Self { + Group { inner } + } + + fn _new_fallback(inner: fallback::Group) -> Self { + Group { + inner: imp::Group::from(inner), + } + } + + /// Creates a new `Group` with the given delimiter and token stream. + /// + /// This constructor will set the span for this group to + /// `Span::call_site()`. To change the span you can use the `set_span` + /// method below. + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + inner: imp::Group::new(delimiter, stream.inner), + } + } + + /// Returns the punctuation used as the delimiter for this group: a set of + /// parentheses, square brackets, or curly braces. + pub fn delimiter(&self) -> Delimiter { + self.inner.delimiter() + } + + /// Returns the `TokenStream` of tokens that are delimited in this `Group`. + /// + /// Note that the returned token stream does not include the delimiter + /// returned above. + pub fn stream(&self) -> TokenStream { + TokenStream::_new(self.inner.stream()) + } + + /// Returns the span for the delimiters of this token stream, spanning the + /// entire `Group`. + /// + /// ```text + /// pub fn span(&self) -> Span { + /// ^^^^^^^ + /// ``` + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Returns the span pointing to the opening delimiter of this group. + /// + /// ```text + /// pub fn span_open(&self) -> Span { + /// ^ + /// ``` + pub fn span_open(&self) -> Span { + Span::_new(self.inner.span_open()) + } + + /// Returns the span pointing to the closing delimiter of this group. + /// + /// ```text + /// pub fn span_close(&self) -> Span { + /// ^ + /// ``` + pub fn span_close(&self) -> Span { + Span::_new(self.inner.span_close()) + } + + /// Returns an object that holds this group's `span_open()` and + /// `span_close()` together (in a more compact representation than holding + /// those 2 spans individually). + pub fn delim_span(&self) -> DelimSpan { + DelimSpan::new(&self.inner) + } + + /// Configures the span for this `Group`'s delimiters, but not its internal + /// tokens. + /// + /// This method will **not** set the span of all the internal tokens spanned + /// by this group, but rather it will only set the span of the delimiter + /// tokens at the level of the `Group`. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +/// Prints the group as a string that should be losslessly convertible back +/// into the same group (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters. +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, formatter) + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, formatter) + } +} + +/// A `Punct` is a single punctuation character like `+`, `-` or `#`. +/// +/// Multicharacter operators like `+=` are represented as two instances of +/// `Punct` with different forms of `Spacing` returned. +#[derive(Clone)] +pub struct Punct { + ch: char, + spacing: Spacing, + span: Span, +} + +/// Whether a `Punct` is followed immediately by another `Punct` or followed by +/// another token or whitespace. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Spacing { + /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`. + Alone, + /// E.g. `+` is `Joint` in `+=` or `'` is `Joint` in `'#`. + /// + /// Additionally, single quote `'` can join with identifiers to form + /// lifetimes `'ident`. + Joint, +} + +impl Punct { + /// Creates a new `Punct` from the given character and spacing. + /// + /// The `ch` argument must be a valid punctuation character permitted by the + /// language, otherwise the function will panic. + /// + /// The returned `Punct` will have the default span of `Span::call_site()` + /// which can be further configured with the `set_span` method below. + pub fn new(ch: char, spacing: Spacing) -> Self { + if let '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' + | '<' | '=' | '>' | '?' | '@' | '^' | '|' | '~' = ch + { + Punct { + ch, + spacing, + span: Span::call_site(), + } + } else { + panic!("unsupported proc macro punctuation character {:?}", ch); + } + } + + /// Returns the value of this punctuation character as `char`. + pub fn as_char(&self) -> char { + self.ch + } + + /// Returns the spacing of this punctuation character, indicating whether + /// it's immediately followed by another `Punct` in the token stream, so + /// they can potentially be combined into a multicharacter operator + /// (`Joint`), or it's followed by some other token or whitespace (`Alone`) + /// so the operator has certainly ended. + pub fn spacing(&self) -> Spacing { + self.spacing + } + + /// Returns the span for this punctuation character. + pub fn span(&self) -> Span { + self.span + } + + /// Configure the span for this punctuation character. + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +/// Prints the punctuation character as a string that should be losslessly +/// convertible back into the same character. +impl Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.ch, f) + } +} + +impl Debug for Punct { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Punct"); + debug.field("char", &self.ch); + debug.field("spacing", &self.spacing); + imp::debug_span_field_if_nontrivial(&mut debug, self.span.inner); + debug.finish() + } +} + +/// A word of Rust code, which may be a keyword or legal variable name. +/// +/// An identifier consists of at least one Unicode code point, the first of +/// which has the XID_Start property and the rest of which have the XID_Continue +/// property. +/// +/// - The empty string is not an identifier. Use `Option`. +/// - A lifetime is not an identifier. Use `syn::Lifetime` instead. +/// +/// An identifier constructed with `Ident::new` is permitted to be a Rust +/// keyword, though parsing one through its [`Parse`] implementation rejects +/// Rust keywords. Use `input.call(Ident::parse_any)` when parsing to match the +/// behaviour of `Ident::new`. +/// +/// [`Parse`]: https://docs.rs/syn/2.0/syn/parse/trait.Parse.html +/// +/// # Examples +/// +/// A new ident can be created from a string using the `Ident::new` function. +/// A span must be provided explicitly which governs the name resolution +/// behavior of the resulting identifier. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// +/// fn main() { +/// let call_ident = Ident::new("calligraphy", Span::call_site()); +/// +/// println!("{}", call_ident); +/// } +/// ``` +/// +/// An ident can be interpolated into a token stream using the `quote!` macro. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// use quote::quote; +/// +/// fn main() { +/// let ident = Ident::new("demo", Span::call_site()); +/// +/// // Create a variable binding whose name is this ident. +/// let expanded = quote! { let #ident = 10; }; +/// +/// // Create a variable binding with a slightly different name. +/// let temp_ident = Ident::new(&format!("new_{}", ident), Span::call_site()); +/// let expanded = quote! { let #temp_ident = 10; }; +/// } +/// ``` +/// +/// A string representation of the ident is available through the `to_string()` +/// method. +/// +/// ``` +/// # use proc_macro2::{Ident, Span}; +/// # +/// # let ident = Ident::new("another_identifier", Span::call_site()); +/// # +/// // Examine the ident as a string. +/// let ident_string = ident.to_string(); +/// if ident_string.len() > 60 { +/// println!("Very long identifier: {}", ident_string) +/// } +/// ``` +#[derive(Clone)] +pub struct Ident { + inner: imp::Ident, + _marker: ProcMacroAutoTraits, +} + +impl Ident { + fn _new(inner: imp::Ident) -> Self { + Ident { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Ident) -> Self { + Ident { + inner: imp::Ident::from(inner), + _marker: MARKER, + } + } + + /// Creates a new `Ident` with the given `string` as well as the specified + /// `span`. + /// + /// The `string` argument must be a valid identifier permitted by the + /// language, otherwise the function will panic. + /// + /// Note that `span`, currently in rustc, configures the hygiene information + /// for this identifier. + /// + /// As of this time `Span::call_site()` explicitly opts-in to "call-site" + /// hygiene meaning that identifiers created with this span will be resolved + /// as if they were written directly at the location of the macro call, and + /// other code at the macro call site will be able to refer to them as well. + /// + /// Later spans like `Span::def_site()` will allow to opt-in to + /// "definition-site" hygiene meaning that identifiers created with this + /// span will be resolved at the location of the macro definition and other + /// code at the macro call site will not be able to refer to them. + /// + /// Due to the current importance of hygiene this constructor, unlike other + /// tokens, requires a `Span` to be specified at construction. + /// + /// # Panics + /// + /// Panics if the input string is neither a keyword nor a legal variable + /// name. If you are not sure whether the string contains an identifier and + /// need to handle an error case, use + /// syn::parse_str::<Ident> + /// rather than `Ident::new`. + #[track_caller] + pub fn new(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_checked(string, span.inner)) + } + + /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). The + /// `string` argument must be a valid identifier permitted by the language + /// (including keywords, e.g. `fn`). Keywords which are usable in path + /// segments (e.g. `self`, `super`) are not supported, and will cause a + /// panic. + #[track_caller] + pub fn new_raw(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_raw_checked(string, span.inner)) + } + + /// Returns the span of this `Ident`. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span of this `Ident`, possibly changing its hygiene + /// context. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.inner == other.inner + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + self.inner == other + } +} + +impl Eq for Ident {} + +impl PartialOrd for Ident { + fn partial_cmp(&self, other: &Ident) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Ident { + fn cmp(&self, other: &Ident) -> Ordering { + self.to_string().cmp(&other.to_string()) + } +} + +impl Hash for Ident { + fn hash(&self, hasher: &mut H) { + self.to_string().hash(hasher); + } +} + +/// Prints the identifier as a string that should be losslessly convertible back +/// into the same identifier. +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A literal string (`"hello"`), byte string (`b"hello"`), character (`'a'`), +/// byte character (`b'a'`), an integer or floating point number with or without +/// a suffix (`1`, `1u8`, `2.3`, `2.3f32`). +/// +/// Boolean literals like `true` and `false` do not belong here, they are +/// `Ident`s. +#[derive(Clone)] +pub struct Literal { + inner: imp::Literal, + _marker: ProcMacroAutoTraits, +} + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new suffixed integer literal with the specified value. + /// + /// This function will create an integer like `1u32` where the integer + /// value specified is the first part of the token and the integral is + /// also suffixed at the end. Literals created from negative numbers may + /// not survive roundtrips through `TokenStream` or strings and may be + /// broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new unsuffixed integer literal with the specified value. + /// + /// This function will create an integer like `1` where the integer + /// value specified is the first part of the token. No suffix is + /// specified on this token, meaning that invocations like + /// `Literal::i8_unsuffixed(1)` are equivalent to + /// `Literal::u32_unsuffixed(1)`. Literals created from negative numbers + /// may not survive roundtrips through `TokenStream` or strings and may + /// be broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +impl Literal { + fn _new(inner: imp::Literal) -> Self { + Literal { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Literal) -> Self { + Literal { + inner: imp::Literal::from(inner), + _marker: MARKER, + } + } + + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + } + + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_unsuffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f64` where the value + /// specified is the preceding part of the token and `f64` is the suffix of + /// the token. This token will always be inferred to be an `f64` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_suffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_suffixed(f)) + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_unsuffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f32` where the value + /// specified is the preceding part of the token and `f32` is the suffix of + /// the token. This token will always be inferred to be an `f32` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_suffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_suffixed(f)) + } + + /// String literal. + pub fn string(string: &str) -> Literal { + Literal::_new(imp::Literal::string(string)) + } + + /// Character literal. + pub fn character(ch: char) -> Literal { + Literal::_new(imp::Literal::character(ch)) + } + + /// Byte character literal. + pub fn byte_character(byte: u8) -> Literal { + Literal::_new(imp::Literal::byte_character(byte)) + } + + /// Byte string literal. + pub fn byte_string(bytes: &[u8]) -> Literal { + Literal::_new(imp::Literal::byte_string(bytes)) + } + + /// C string literal. + pub fn c_string(string: &CStr) -> Literal { + Literal::_new(imp::Literal::c_string(string)) + } + + /// Returns the span encompassing this literal. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span associated for this literal. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } + + /// Returns a `Span` that is a subset of `self.span()` containing only + /// the source bytes in range `range`. Returns `None` if the would-be + /// trimmed span is outside the bounds of `self`. + /// + /// Warning: the underlying [`proc_macro::Literal::subspan`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn subspan>(&self, range: R) -> Option { + self.inner.subspan(range).map(Span::_new) + } + + // Intended for the `quote!` macro to use when constructing a proc-macro2 + // token out of a macro_rules $:literal token, which is already known to be + // a valid literal. This avoids reparsing/validating the literal's string + // representation. This is not public API other than for quote. + #[doc(hidden)] + pub unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(unsafe { imp::Literal::from_str_unchecked(repr) }) + } +} + +impl FromStr for Literal { + type Err = LexError; + + fn from_str(repr: &str) -> Result { + match imp::Literal::from_str_checked(repr) { + Ok(lit) => Ok(Literal::_new(lit)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +pub mod token_stream { + use crate::marker::{ProcMacroAutoTraits, MARKER}; + use crate::{imp, TokenTree}; + use core::fmt::{self, Debug}; + + pub use crate::TokenStream; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// + /// The iteration is "shallow", e.g. the iterator doesn't recurse into + /// delimited groups, and returns whole groups as token trees. + #[derive(Clone)] + pub struct IntoIter { + inner: imp::TokenTreeIter, + _marker: ProcMacroAutoTraits, + } + + impl Iterator for IntoIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + } + + impl Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } + } + + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { + inner: self.inner.into_iter(), + _marker: MARKER, + } + } + } +} diff --git a/rust/proc-macro2/location.rs b/rust/proc-macro2/location.rs new file mode 100644 index 000000000..3a11871e2 --- /dev/null +++ b/rust/proc-macro2/location.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use core::cmp::Ordering; + +/// A line-column pair representing the start or end of a `Span`. +/// +/// This type is semver exempt and not exposed by default. +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct LineColumn { + /// The 1-indexed line in the source file on which the span starts or ends + /// (inclusive). + pub line: usize, + /// The 0-indexed column (in UTF-8 characters) in the source file on which + /// the span starts or ends (inclusive). + pub column: usize, +} + +impl Ord for LineColumn { + fn cmp(&self, other: &Self) -> Ordering { + self.line + .cmp(&other.line) + .then(self.column.cmp(&other.column)) + } +} + +impl PartialOrd for LineColumn { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/rust/proc-macro2/marker.rs b/rust/proc-macro2/marker.rs new file mode 100644 index 000000000..6f3224248 --- /dev/null +++ b/rust/proc-macro2/marker.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::rc::Rc; +use core::marker::PhantomData; +use core::panic::{RefUnwindSafe, UnwindSafe}; + +// Zero sized marker with the correct set of autotrait impls we want all proc +// macro types to have. +#[derive(Copy, Clone)] +#[cfg_attr( + all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)), + derive(PartialEq, Eq) +)] +pub(crate) struct ProcMacroAutoTraits(PhantomData>); + +pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData); + +impl UnwindSafe for ProcMacroAutoTraits {} +impl RefUnwindSafe for ProcMacroAutoTraits {} diff --git a/rust/proc-macro2/parse.rs b/rust/proc-macro2/parse.rs new file mode 100644 index 000000000..a005dea1f --- /dev/null +++ b/rust/proc-macro2/parse.rs @@ -0,0 +1,997 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::fallback::{ + self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream, + TokenStreamBuilder, +}; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::char; +use core::str::{Bytes, CharIndices, Chars}; + +#[derive(Copy, Clone, Eq, PartialEq)] +pub(crate) struct Cursor<'a> { + pub(crate) rest: &'a str, + #[cfg(span_locations)] + pub(crate) off: u32, +} + +impl<'a> Cursor<'a> { + pub(crate) fn advance(&self, bytes: usize) -> Cursor<'a> { + let (_front, rest) = self.rest.split_at(bytes); + Cursor { + rest, + #[cfg(span_locations)] + off: self.off + _front.chars().count() as u32, + } + } + + pub(crate) fn starts_with(&self, s: &str) -> bool { + self.rest.starts_with(s) + } + + pub(crate) fn starts_with_char(&self, ch: char) -> bool { + self.rest.starts_with(ch) + } + + pub(crate) fn starts_with_fn(&self, f: Pattern) -> bool + where + Pattern: FnMut(char) -> bool, + { + self.rest.starts_with(f) + } + + pub(crate) fn is_empty(&self) -> bool { + self.rest.is_empty() + } + + fn len(&self) -> usize { + self.rest.len() + } + + fn as_bytes(&self) -> &'a [u8] { + self.rest.as_bytes() + } + + fn bytes(&self) -> Bytes<'a> { + self.rest.bytes() + } + + fn chars(&self) -> Chars<'a> { + self.rest.chars() + } + + fn char_indices(&self) -> CharIndices<'a> { + self.rest.char_indices() + } + + fn parse(&self, tag: &str) -> Result, Reject> { + if self.starts_with(tag) { + Ok(self.advance(tag.len())) + } else { + Err(Reject) + } + } +} + +pub(crate) struct Reject; +type PResult<'a, O> = Result<(Cursor<'a>, O), Reject>; + +fn skip_whitespace(input: Cursor) -> Cursor { + let mut s = input; + + while !s.is_empty() { + let byte = s.as_bytes()[0]; + if byte == b'/' { + if s.starts_with("//") + && (!s.starts_with("///") || s.starts_with("////")) + && !s.starts_with("//!") + { + let (cursor, _) = take_until_newline_or_eof(s); + s = cursor; + continue; + } else if s.starts_with("/**/") { + s = s.advance(4); + continue; + } else if s.starts_with("/*") + && (!s.starts_with("/**") || s.starts_with("/***")) + && !s.starts_with("/*!") + { + match block_comment(s) { + Ok((rest, _)) => { + s = rest; + continue; + } + Err(Reject) => return s, + } + } + } + match byte { + b' ' | 0x09..=0x0d => { + s = s.advance(1); + continue; + } + b if b.is_ascii() => {} + _ => { + let ch = s.chars().next().unwrap(); + if is_whitespace(ch) { + s = s.advance(ch.len_utf8()); + continue; + } + } + } + return s; + } + s +} + +fn block_comment(input: Cursor) -> PResult<&str> { + if !input.starts_with("/*") { + return Err(Reject); + } + + let mut depth = 0usize; + let bytes = input.as_bytes(); + let mut i = 0usize; + let upper = bytes.len() - 1; + + while i < upper { + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + depth += 1; + i += 1; // eat '*' + } else if bytes[i] == b'*' && bytes[i + 1] == b'/' { + depth -= 1; + if depth == 0 { + return Ok((input.advance(i + 2), &input.rest[..i + 2])); + } + i += 1; // eat '/' + } + i += 1; + } + + Err(Reject) +} + +fn is_whitespace(ch: char) -> bool { + // Rust treats left-to-right mark and right-to-left mark as whitespace + ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' +} + +fn word_break(input: Cursor) -> Result { + match input.chars().next() { + Some(ch) if is_ident_continue(ch) => Err(Reject), + Some(_) | None => Ok(input), + } +} + +// Rustc's representation of a macro expansion error in expression position or +// type position. +const ERROR: &str = "(/*ERROR*/)"; + +pub(crate) fn token_stream(mut input: Cursor) -> Result { + let mut trees = TokenStreamBuilder::new(); + let mut stack = Vec::new(); + + loop { + input = skip_whitespace(input); + + if let Ok((rest, ())) = doc_comment(input, &mut trees) { + input = rest; + continue; + } + + #[cfg(span_locations)] + let lo = input.off; + + let first = match input.bytes().next() { + Some(first) => first, + None => match stack.last() { + None => return Ok(trees.build()), + #[cfg(span_locations)] + Some((lo, _frame)) => { + return Err(LexError { + span: Span { lo: *lo, hi: *lo }, + }) + } + #[cfg(not(span_locations))] + Some(_frame) => return Err(LexError { span: Span {} }), + }, + }; + + if let Some(open_delimiter) = match first { + b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis), + b'[' => Some(Delimiter::Bracket), + b'{' => Some(Delimiter::Brace), + _ => None, + } { + input = input.advance(1); + let frame = (open_delimiter, trees); + #[cfg(span_locations)] + let frame = (lo, frame); + stack.push(frame); + trees = TokenStreamBuilder::new(); + } else if let Some(close_delimiter) = match first { + b')' => Some(Delimiter::Parenthesis), + b']' => Some(Delimiter::Bracket), + b'}' => Some(Delimiter::Brace), + _ => None, + } { + let frame = match stack.pop() { + Some(frame) => frame, + None => return Err(lex_error(input)), + }; + #[cfg(span_locations)] + let (lo, frame) = frame; + let (open_delimiter, outer) = frame; + if open_delimiter != close_delimiter { + return Err(lex_error(input)); + } + input = input.advance(1); + let mut g = Group::new(open_delimiter, trees.build()); + g.set_span(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: input.off, + }); + trees = outer; + trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g))); + } else { + let (rest, mut tt) = match leaf_token(input) { + Ok((rest, tt)) => (rest, tt), + Err(Reject) => return Err(lex_error(input)), + }; + tt.set_span(crate::Span::_new_fallback(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + })); + trees.push_token_from_parser(tt); + input = rest; + } + } +} + +fn lex_error(cursor: Cursor) -> LexError { + #[cfg(not(span_locations))] + let _ = cursor; + LexError { + span: Span { + #[cfg(span_locations)] + lo: cursor.off, + #[cfg(span_locations)] + hi: cursor.off, + }, + } +} + +fn leaf_token(input: Cursor) -> PResult { + if let Ok((input, l)) = literal(input) { + // must be parsed before ident + Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l)))) + } else if let Ok((input, p)) = punct(input) { + Ok((input, TokenTree::Punct(p))) + } else if let Ok((input, i)) = ident(input) { + Ok((input, TokenTree::Ident(i))) + } else if input.starts_with(ERROR) { + let rest = input.advance(ERROR.len()); + let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned())); + Ok((rest, TokenTree::Literal(repr))) + } else { + Err(Reject) + } +} + +fn ident(input: Cursor) -> PResult { + if [ + "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#", + ] + .iter() + .any(|prefix| input.starts_with(prefix)) + { + Err(Reject) + } else { + ident_any(input) + } +} + +fn ident_any(input: Cursor) -> PResult { + let raw = input.starts_with("r#"); + let rest = input.advance((raw as usize) << 1); + + let (rest, sym) = ident_not_raw(rest)?; + + if !raw { + let ident = + crate::Ident::_new_fallback(Ident::new_unchecked(sym, fallback::Span::call_site())); + return Ok((rest, ident)); + } + + match sym { + "_" | "super" | "self" | "Self" | "crate" => return Err(Reject), + _ => {} + } + + let ident = + crate::Ident::_new_fallback(Ident::new_raw_unchecked(sym, fallback::Span::call_site())); + Ok((rest, ident)) +} + +fn ident_not_raw(input: Cursor) -> PResult<&str> { + let mut chars = input.char_indices(); + + match chars.next() { + Some((_, ch)) if is_ident_start(ch) => {} + _ => return Err(Reject), + } + + let mut end = input.len(); + for (i, ch) in chars { + if !is_ident_continue(ch) { + end = i; + break; + } + } + + Ok((input.advance(end), &input.rest[..end])) +} + +pub(crate) fn literal(input: Cursor) -> PResult { + let rest = literal_nocapture(input)?; + let end = input.len() - rest.len(); + Ok((rest, Literal::_new(input.rest[..end].to_string()))) +} + +fn literal_nocapture(input: Cursor) -> Result { + if let Ok(ok) = string(input) { + Ok(ok) + } else if let Ok(ok) = byte_string(input) { + Ok(ok) + } else if let Ok(ok) = c_string(input) { + Ok(ok) + } else if let Ok(ok) = byte(input) { + Ok(ok) + } else if let Ok(ok) = character(input) { + Ok(ok) + } else if let Ok(ok) = float(input) { + Ok(ok) + } else if let Ok(ok) = int(input) { + Ok(ok) + } else { + Err(Reject) + } +} + +fn literal_suffix(input: Cursor) -> Cursor { + match ident_not_raw(input) { + Ok((input, _)) => input, + Err(Reject) => input, + } +} + +fn string(input: Cursor) -> Result { + if let Ok(input) = input.parse("\"") { + cooked_string(input) + } else if let Ok(input) = input.parse("r") { + raw_string(input) + } else { + Err(Reject) + } +} + +fn cooked_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_char(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {} + Some((_, 'u')) => { + backslash_u(&mut chars)?; + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + _ch => {} + } + } + Err(Reject) +} + +fn raw_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + _ => {} + } + } + Err(Reject) +} + +fn byte_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("b\"") { + cooked_byte_string(input) + } else if let Ok(input) = input.parse("br") { + raw_byte_string(input) + } else { + Err(Reject) + } +} + +fn cooked_byte_string(mut input: Cursor) -> Result { + let mut bytes = input.bytes().enumerate(); + while let Some((offset, b)) = bytes.next() { + match b { + b'"' => { + let input = input.advance(offset + 1); + return Ok(literal_suffix(input)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\\' => match bytes.next() { + Some((_, b'x')) => { + backslash_x_byte(&mut bytes)?; + } + Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {} + Some((newline, b @ (b'\n' | b'\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, b)?; + bytes = input.bytes().enumerate(); + } + _ => break, + }, + b if b.is_ascii() => {} + _ => break, + } + } + Err(Reject) +} + +fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> { + for (i, byte) in input.bytes().enumerate() { + match byte { + b'"' => { + if i > 255 { + // https://github.com/rust-lang/rust/pull/95251 + return Err(Reject); + } + return Ok((input.advance(i + 1), &input.rest[..i])); + } + b'#' => {} + _ => break, + } + } + Err(Reject) +} + +fn raw_byte_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + other => { + if !other.is_ascii() { + break; + } + } + } + } + Err(Reject) +} + +fn c_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("c\"") { + cooked_c_string(input) + } else if let Ok(input) = input.parse("cr") { + raw_c_string(input) + } else { + Err(Reject) + } +} + +fn raw_c_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\0' => break, + _ => {} + } + } + Err(Reject) +} + +fn cooked_c_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_nonzero(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {} + Some((_, 'u')) => { + if backslash_u(&mut chars)? == '\0' { + break; + } + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + '\0' => break, + _ch => {} + } + } + Err(Reject) +} + +fn byte(input: Cursor) -> Result { + let input = input.parse("b'")?; + let mut bytes = input.bytes().enumerate(); + let ok = match bytes.next().map(|(_, b)| b) { + Some(b'\\') => match bytes.next().map(|(_, b)| b) { + Some(b'x') => backslash_x_byte(&mut bytes).is_ok(), + Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true, + _ => false, + }, + b => b.is_some(), + }; + if !ok { + return Err(Reject); + } + let (offset, _) = bytes.next().ok_or(Reject)?; + if !input.chars().as_str().is_char_boundary(offset) { + return Err(Reject); + } + let input = input.advance(offset).parse("'")?; + Ok(literal_suffix(input)) +} + +fn character(input: Cursor) -> Result { + let input = input.parse("'")?; + let mut chars = input.char_indices(); + let ok = match chars.next().map(|(_, ch)| ch) { + Some('\\') => match chars.next().map(|(_, ch)| ch) { + Some('x') => backslash_x_char(&mut chars).is_ok(), + Some('u') => backslash_u(&mut chars).is_ok(), + Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true, + _ => false, + }, + ch => ch.is_some(), + }; + if !ok { + return Err(Reject); + } + let (idx, _) = chars.next().ok_or(Reject)?; + let input = input.advance(idx).parse("'")?; + Ok(literal_suffix(input)) +} + +macro_rules! next_ch { + ($chars:ident @ $pat:pat) => { + match $chars.next() { + Some((_, ch)) => match ch { + $pat => ch, + _ => return Err(Reject), + }, + None => return Err(Reject), + } + }; +} + +fn backslash_x_char(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ '0'..='7'); + next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + Ok(()) +} + +fn backslash_x_byte(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + Ok(()) +} + +fn backslash_x_nonzero(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + if first == '0' && second == '0' { + Err(Reject) + } else { + Ok(()) + } +} + +fn backslash_u(chars: &mut I) -> Result +where + I: Iterator, +{ + next_ch!(chars @ '{'); + let mut value = 0; + let mut len = 0; + for (_, ch) in chars { + let digit = match ch { + '0'..='9' => ch as u8 - b'0', + 'a'..='f' => 10 + ch as u8 - b'a', + 'A'..='F' => 10 + ch as u8 - b'A', + '_' if len > 0 => continue, + '}' if len > 0 => return char::from_u32(value).ok_or(Reject), + _ => break, + }; + if len == 6 { + break; + } + value *= 0x10; + value += u32::from(digit); + len += 1; + } + Err(Reject) +} + +fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> { + let mut whitespace = input.bytes().enumerate(); + loop { + if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') { + return Err(Reject); + } + match whitespace.next() { + Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => { + last = b; + } + Some((offset, _)) => { + *input = input.advance(offset); + return Ok(()); + } + None => return Err(Reject), + } + } +} + +fn float(input: Cursor) -> Result { + let mut rest = float_digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn float_digits(input: Cursor) -> Result { + let mut chars = input.chars().peekable(); + match chars.next() { + Some(ch) if '0' <= ch && ch <= '9' => {} + _ => return Err(Reject), + } + + let mut len = 1; + let mut has_dot = false; + let mut has_exp = false; + while let Some(&ch) = chars.peek() { + match ch { + '0'..='9' | '_' => { + chars.next(); + len += 1; + } + '.' => { + if has_dot { + break; + } + chars.next(); + if chars + .peek() + .map_or(false, |&ch| ch == '.' || is_ident_start(ch)) + { + return Err(Reject); + } + len += 1; + has_dot = true; + } + 'e' | 'E' => { + chars.next(); + len += 1; + has_exp = true; + break; + } + _ => break, + } + } + + if !(has_dot || has_exp) { + return Err(Reject); + } + + if has_exp { + let token_before_exp = if has_dot { + Ok(input.advance(len - 1)) + } else { + Err(Reject) + }; + let mut has_sign = false; + let mut has_exp_value = false; + while let Some(&ch) = chars.peek() { + match ch { + '+' | '-' => { + if has_exp_value { + break; + } + if has_sign { + return token_before_exp; + } + chars.next(); + len += 1; + has_sign = true; + } + '0'..='9' => { + chars.next(); + len += 1; + has_exp_value = true; + } + '_' => { + chars.next(); + len += 1; + } + _ => break, + } + } + if !has_exp_value { + return token_before_exp; + } + } + + Ok(input.advance(len)) +} + +fn int(input: Cursor) -> Result { + let mut rest = digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn digits(mut input: Cursor) -> Result { + let base = if input.starts_with("0x") { + input = input.advance(2); + 16 + } else if input.starts_with("0o") { + input = input.advance(2); + 8 + } else if input.starts_with("0b") { + input = input.advance(2); + 2 + } else { + 10 + }; + + let mut len = 0; + let mut empty = true; + for b in input.bytes() { + match b { + b'0'..=b'9' => { + let digit = (b - b'0') as u64; + if digit >= base { + return Err(Reject); + } + } + b'a'..=b'f' => { + let digit = 10 + (b - b'a') as u64; + if digit >= base { + break; + } + } + b'A'..=b'F' => { + let digit = 10 + (b - b'A') as u64; + if digit >= base { + break; + } + } + b'_' => { + if empty && base == 10 { + return Err(Reject); + } + len += 1; + continue; + } + _ => break, + } + len += 1; + empty = false; + } + if empty { + Err(Reject) + } else { + Ok(input.advance(len)) + } +} + +fn punct(input: Cursor) -> PResult { + let (rest, ch) = punct_char(input)?; + if ch == '\'' { + let (after_lifetime, _ident) = ident_any(rest)?; + if after_lifetime.starts_with_char('\'') + || (after_lifetime.starts_with_char('#') && !rest.starts_with("r#")) + { + Err(Reject) + } else { + Ok((rest, Punct::new('\'', Spacing::Joint))) + } + } else { + let kind = match punct_char(rest) { + Ok(_) => Spacing::Joint, + Err(Reject) => Spacing::Alone, + }; + Ok((rest, Punct::new(ch, kind))) + } +} + +fn punct_char(input: Cursor) -> PResult { + if input.starts_with("//") || input.starts_with("/*") { + // Do not accept `/` of a comment as a punct. + return Err(Reject); + } + + let mut chars = input.chars(); + let first = match chars.next() { + Some(ch) => ch, + None => { + return Err(Reject); + } + }; + let recognized = "~!@#$%^&*-=+|;:,<.>/?'"; + if recognized.contains(first) { + Ok((input.advance(first.len_utf8()), first)) + } else { + Err(Reject) + } +} + +fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> { + #[cfg(span_locations)] + let lo = input.off; + let (rest, (comment, inner)) = doc_comment_contents(input)?; + let fallback_span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + let span = crate::Span::_new_fallback(fallback_span); + + let mut scan_for_bare_cr = comment; + while let Some(cr) = scan_for_bare_cr.find('\r') { + let rest = &scan_for_bare_cr[cr + 1..]; + if !rest.starts_with('\n') { + return Err(Reject); + } + scan_for_bare_cr = rest; + } + + let mut pound = Punct::new('#', Spacing::Alone); + pound.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(pound)); + + if inner { + let mut bang = Punct::new('!', Spacing::Alone); + bang.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(bang)); + } + + let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span)); + let mut equal = Punct::new('=', Spacing::Alone); + equal.set_span(span); + let mut literal = crate::Literal::_new_fallback(Literal::string(comment)); + literal.set_span(span); + let mut bracketed = TokenStreamBuilder::with_capacity(3); + bracketed.push_token_from_parser(TokenTree::Ident(doc_ident)); + bracketed.push_token_from_parser(TokenTree::Punct(equal)); + bracketed.push_token_from_parser(TokenTree::Literal(literal)); + let group = Group::new(Delimiter::Bracket, bracketed.build()); + let mut group = crate::Group::_new_fallback(group); + group.set_span(span); + trees.push_token_from_parser(TokenTree::Group(group)); + + Ok((rest, ())) +} + +fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> { + if input.starts_with("//!") { + let input = input.advance(3); + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, true))) + } else if input.starts_with("/*!") { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], true))) + } else if input.starts_with("///") { + let input = input.advance(3); + if input.starts_with_char('/') { + return Err(Reject); + } + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, false))) + } else if input.starts_with("/**") && !input.rest[3..].starts_with('*') { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], false))) + } else { + Err(Reject) + } +} + +fn take_until_newline_or_eof(input: Cursor) -> (Cursor, &str) { + let chars = input.char_indices(); + + for (i, ch) in chars { + if ch == '\n' { + return (input.advance(i), &input.rest[..i]); + } else if ch == '\r' && input.rest[i + 1..].starts_with('\n') { + return (input.advance(i + 1), &input.rest[..i]); + } + } + + (input.advance(input.len()), input.rest) +} diff --git a/rust/proc-macro2/probe.rs b/rust/proc-macro2/probe.rs new file mode 100644 index 000000000..d68aa8cfd --- /dev/null +++ b/rust/proc-macro2/probe.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(dead_code)] + +#[cfg(proc_macro_span)] +pub(crate) mod proc_macro_span; + +#[cfg(proc_macro_span_file)] +pub(crate) mod proc_macro_span_file; + +#[cfg(proc_macro_span_location)] +pub(crate) mod proc_macro_span_location; diff --git a/rust/proc-macro2/probe/proc_macro_span.rs b/rust/proc-macro2/probe/proc_macro_span.rs new file mode 100644 index 000000000..892a7eb3e --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// This code exercises the surface area that we expect of Span's unstable API. +// If the current toolchain is able to compile it, then proc-macro2 is able to +// offer these APIs too. + +#![cfg_attr(procmacro2_build_probe, feature(proc_macro_span))] + +extern crate proc_macro; + +use core::ops::{Range, RangeBounds}; +use proc_macro::{Literal, Span}; +use std::path::PathBuf; + +pub fn byte_range(this: &Span) -> Range { + this.byte_range() +} + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} + +pub fn join(this: &Span, other: Span) -> Option { + this.join(other) +} + +pub fn subspan>(this: &Literal, range: R) -> Option { + this.subspan(range) +} + +// Include in sccache cache key. +#[cfg(procmacro2_build_probe)] +const _: Option<&str> = option_env!("RUSTC_BOOTSTRAP"); diff --git a/rust/proc-macro2/probe/proc_macro_span_file.rs b/rust/proc-macro2/probe/proc_macro_span_file.rs new file mode 100644 index 000000000..f2dbc4056 --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_file.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; +use std::path::PathBuf; + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} diff --git a/rust/proc-macro2/probe/proc_macro_span_location.rs b/rust/proc-macro2/probe/proc_macro_span_location.rs new file mode 100644 index 000000000..ae19c9339 --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_location.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} diff --git a/rust/proc-macro2/rcvec.rs b/rust/proc-macro2/rcvec.rs new file mode 100644 index 000000000..e224ebe27 --- /dev/null +++ b/rust/proc-macro2/rcvec.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::rc::Rc; +use alloc::vec; +use core::mem; +use core::panic::RefUnwindSafe; +use core::slice; + +pub(crate) struct RcVec { + inner: Rc>, +} + +pub(crate) struct RcVecBuilder { + inner: Vec, +} + +pub(crate) struct RcVecMut<'a, T> { + inner: &'a mut Vec, +} + +#[derive(Clone)] +pub(crate) struct RcVecIntoIter { + inner: vec::IntoIter, +} + +impl RcVec { + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.inner.len() + } + + pub(crate) fn iter(&self) -> slice::Iter { + self.inner.iter() + } + + pub(crate) fn make_mut(&mut self) -> RcVecMut + where + T: Clone, + { + RcVecMut { + inner: Rc::make_mut(&mut self.inner), + } + } + + pub(crate) fn get_mut(&mut self) -> Option> { + let inner = Rc::get_mut(&mut self.inner)?; + Some(RcVecMut { inner }) + } + + pub(crate) fn make_owned(mut self) -> RcVecBuilder + where + T: Clone, + { + let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) { + mem::take(owned) + } else { + Vec::clone(&self.inner) + }; + RcVecBuilder { inner: vec } + } +} + +impl RcVecBuilder { + pub(crate) fn new() -> Self { + RcVecBuilder { inner: Vec::new() } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + RcVecBuilder { + inner: Vec::with_capacity(cap), + } + } + + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { + inner: &mut self.inner, + } + } + + pub(crate) fn build(self) -> RcVec { + RcVec { + inner: Rc::new(self.inner), + } + } +} + +impl<'a, T> RcVecMut<'a, T> { + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { inner: self.inner } + } + + pub(crate) fn take(self) -> RcVecBuilder { + let vec = mem::take(self.inner); + RcVecBuilder { inner: vec } + } +} + +impl Clone for RcVec { + fn clone(&self) -> Self { + RcVec { + inner: Rc::clone(&self.inner), + } + } +} + +impl IntoIterator for RcVecBuilder { + type Item = T; + type IntoIter = RcVecIntoIter; + + fn into_iter(self) -> Self::IntoIter { + RcVecIntoIter { + inner: self.inner.into_iter(), + } + } +} + +impl Iterator for RcVecIntoIter { + type Item = T; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl RefUnwindSafe for RcVec where T: RefUnwindSafe {} diff --git a/rust/proc-macro2/wrapper.rs b/rust/proc-macro2/wrapper.rs new file mode 100644 index 000000000..6792e9834 --- /dev/null +++ b/rust/proc-macro2/wrapper.rs @@ -0,0 +1,986 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::detection::inside_proc_macro; +use crate::fallback::{self, FromStr2 as _}; +#[cfg(span_locations)] +use crate::location::LineColumn; +#[cfg(proc_macro_span)] +use crate::probe::proc_macro_span; +#[cfg(all(span_locations, proc_macro_span_file))] +use crate::probe::proc_macro_span_file; +#[cfg(all(span_locations, proc_macro_span_location))] +use crate::probe::proc_macro_span_location; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::fmt::{self, Debug, Display}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[derive(Clone)] +pub(crate) enum TokenStream { + Compiler(DeferredTokenStream), + Fallback(fallback::TokenStream), +} + +// Work around https://github.com/rust-lang/rust/issues/65080. +// In `impl Extend for TokenStream` which is used heavily by quote, +// we hold on to the appended tokens and do proc_macro::TokenStream::extend as +// late as possible to batch together consecutive uses of the Extend impl. +#[derive(Clone)] +pub(crate) struct DeferredTokenStream { + stream: proc_macro::TokenStream, + extra: Vec, +} + +pub(crate) enum LexError { + Compiler(proc_macro::LexError), + Fallback(fallback::LexError), + + // Rustc was supposed to return a LexError, but it panicked instead. + // https://github.com/rust-lang/rust/issues/58736 + CompilerPanic, +} + +#[cold] +fn mismatch(line: u32) -> ! { + #[cfg(procmacro2_backtrace)] + { + let backtrace = std::backtrace::Backtrace::force_capture(); + panic!("compiler/fallback mismatch L{}\n\n{}", line, backtrace) + } + #[cfg(not(procmacro2_backtrace))] + { + panic!("compiler/fallback mismatch L{}", line) + } +} + +impl DeferredTokenStream { + fn new(stream: proc_macro::TokenStream) -> Self { + DeferredTokenStream { + stream, + extra: Vec::new(), + } + } + + fn is_empty(&self) -> bool { + self.stream.is_empty() && self.extra.is_empty() + } + + fn evaluate_now(&mut self) { + // If-check provides a fast short circuit for the common case of `extra` + // being empty, which saves a round trip over the proc macro bridge. + // Improves macro expansion time in winrt by 6% in debug mode. + if !self.extra.is_empty() { + self.stream.extend(self.extra.drain(..)); + } + } + + fn into_token_stream(mut self) -> proc_macro::TokenStream { + self.evaluate_now(); + self.stream + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new())) + } else { + TokenStream::Fallback(fallback::TokenStream::new()) + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + if inside_proc_macro() { + Ok(TokenStream::Compiler(DeferredTokenStream::new( + proc_macro::TokenStream::from_str_checked(src)?, + ))) + } else { + Ok(TokenStream::Fallback( + fallback::TokenStream::from_str_checked(src)?, + )) + } + } + + pub(crate) fn is_empty(&self) -> bool { + match self { + TokenStream::Compiler(tts) => tts.is_empty(), + TokenStream::Fallback(tts) => tts.is_empty(), + } + } + + fn unwrap_nightly(self) -> proc_macro::TokenStream { + match self { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + } + } + + fn unwrap_stable(self) -> fallback::TokenStream { + match self { + TokenStream::Compiler(_) => mismatch(line!()), + TokenStream::Fallback(s) => s, + } + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Display::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Display::fmt(tts, f), + } + } +} + +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::Compiler(DeferredTokenStream::new(inner)) + } +} + +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + match inner { + TokenStream::Compiler(inner) => inner.into_token_stream(), + TokenStream::Fallback(inner) => { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } + } + } +} + +impl From for TokenStream { + fn from(inner: fallback::TokenStream) -> Self { + TokenStream::Fallback(inner) + } +} + +// Assumes inside_proc_macro(). +fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree { + match token { + TokenTree::Group(tt) => proc_macro::TokenTree::Group(tt.inner.unwrap_nightly()), + TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + Spacing::Joint => proc_macro::Spacing::Joint, + Spacing::Alone => proc_macro::Spacing::Alone, + }; + let mut punct = proc_macro::Punct::new(tt.as_char(), spacing); + punct.set_span(tt.span().inner.unwrap_nightly()); + proc_macro::TokenTree::Punct(punct) + } + TokenTree::Ident(tt) => proc_macro::TokenTree::Ident(tt.inner.unwrap_nightly()), + TokenTree::Literal(tt) => proc_macro::TokenTree::Literal(tt.inner.unwrap_nightly()), + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::from( + into_compiler_token(token), + ))) + } else { + TokenStream::Fallback(fallback::TokenStream::from(token)) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new( + trees.into_iter().map(into_compiler_token).collect(), + )) + } else { + TokenStream::Fallback(trees.into_iter().collect()) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut streams = streams.into_iter(); + match streams.next() { + Some(TokenStream::Compiler(mut first)) => { + first.evaluate_now(); + first.stream.extend(streams.map(|s| match s { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + })); + TokenStream::Compiler(first) + } + Some(TokenStream::Fallback(mut first)) => { + first.extend(streams.map(|s| match s { + TokenStream::Fallback(s) => s, + TokenStream::Compiler(_) => mismatch(line!()), + })); + TokenStream::Fallback(first) + } + None => TokenStream::new(), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, stream: I) { + match self { + TokenStream::Compiler(tts) => { + // Here is the reason for DeferredTokenStream. + for token in stream { + tts.extra.push(into_compiler_token(token)); + } + } + TokenStream::Fallback(tts) => tts.extend(stream), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + match self { + TokenStream::Compiler(tts) => { + tts.evaluate_now(); + tts.stream + .extend(streams.into_iter().map(TokenStream::unwrap_nightly)); + } + TokenStream::Fallback(tts) => { + tts.extend(streams.into_iter().map(TokenStream::unwrap_stable)); + } + } + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Debug::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Debug::fmt(tts, f), + } + } +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + match self { + LexError::Compiler(_) | LexError::CompilerPanic => Span::call_site(), + LexError::Fallback(e) => Span::Fallback(e.span()), + } + } +} + +impl From for LexError { + fn from(e: proc_macro::LexError) -> Self { + LexError::Compiler(e) + } +} + +impl From for LexError { + fn from(e: fallback::LexError) -> Self { + LexError::Fallback(e) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Debug::fmt(e, f), + LexError::Fallback(e) => Debug::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Debug::fmt(&fallback, f) + } + } + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Display::fmt(e, f), + LexError::Fallback(e) => Display::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Display::fmt(&fallback, f) + } + } + } +} + +#[derive(Clone)] +pub(crate) enum TokenTreeIter { + Compiler(proc_macro::token_stream::IntoIter), + Fallback(fallback::TokenTreeIter), +} + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + match self { + TokenStream::Compiler(tts) => { + TokenTreeIter::Compiler(tts.into_token_stream().into_iter()) + } + TokenStream::Fallback(tts) => TokenTreeIter::Fallback(tts.into_iter()), + } + } +} + +impl Iterator for TokenTreeIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + let token = match self { + TokenTreeIter::Compiler(iter) => iter.next()?, + TokenTreeIter::Fallback(iter) => return iter.next(), + }; + Some(match token { + proc_macro::TokenTree::Group(tt) => { + TokenTree::Group(crate::Group::_new(Group::Compiler(tt))) + } + proc_macro::TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + proc_macro::Spacing::Joint => Spacing::Joint, + proc_macro::Spacing::Alone => Spacing::Alone, + }; + let mut o = Punct::new(tt.as_char(), spacing); + o.set_span(crate::Span::_new(Span::Compiler(tt.span()))); + TokenTree::Punct(o) + } + proc_macro::TokenTree::Ident(s) => { + TokenTree::Ident(crate::Ident::_new(Ident::Compiler(s))) + } + proc_macro::TokenTree::Literal(l) => { + TokenTree::Literal(crate::Literal::_new(Literal::Compiler(l))) + } + }) + } + + fn size_hint(&self) -> (usize, Option) { + match self { + TokenTreeIter::Compiler(tts) => tts.size_hint(), + TokenTreeIter::Fallback(tts) => tts.size_hint(), + } + } +} + +#[derive(Copy, Clone)] +pub(crate) enum Span { + Compiler(proc_macro::Span), + Fallback(fallback::Span), +} + +impl Span { + pub(crate) fn call_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::call_site()) + } else { + Span::Fallback(fallback::Span::call_site()) + } + } + + pub(crate) fn mixed_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::mixed_site()) + } else { + Span::Fallback(fallback::Span::mixed_site()) + } + } + + #[cfg(super_unstable)] + pub(crate) fn def_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::def_site()) + } else { + Span::Fallback(fallback::Span::def_site()) + } + } + + pub(crate) fn resolved_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn unwrap(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => panic!("proc_macro::Span is only available in procedural macros"), + } + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => proc_macro_span::byte_range(s), + #[cfg(not(proc_macro_span))] + Span::Compiler(_) => 0..0, + Span::Fallback(s) => s.byte_range(), + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => LineColumn { + line: proc_macro_span_location::line(s), + column: proc_macro_span_location::column(s).saturating_sub(1), + }, + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.start(), + } + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => { + let end = proc_macro_span_location::end(s); + LineColumn { + line: proc_macro_span_location::line(&end), + column: proc_macro_span_location::column(&end).saturating_sub(1), + } + } + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.end(), + } + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => "".to_owned(), + Span::Fallback(s) => s.file(), + } + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::local_file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => None, + Span::Fallback(s) => s.local_file(), + } + } + + pub(crate) fn join(&self, other: Span) -> Option { + let ret = match (self, other) { + #[cfg(proc_macro_span)] + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(proc_macro_span::join(a, b)?), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.join(b)?), + _ => return None, + }; + Some(ret) + } + + #[cfg(super_unstable)] + pub(crate) fn eq(&self, other: &Span) -> bool { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => a.eq(b), + (Span::Fallback(a), Span::Fallback(b)) => a.eq(b), + _ => false, + } + } + + pub(crate) fn source_text(&self) -> Option { + match self { + #[cfg(not(no_source_text))] + Span::Compiler(s) => s.source_text(), + #[cfg(no_source_text)] + Span::Compiler(_) => None, + Span::Fallback(s) => s.source_text(), + } + } + + fn unwrap_nightly(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for crate::Span { + fn from(proc_span: proc_macro::Span) -> Self { + crate::Span::_new(Span::Compiler(proc_span)) + } +} + +impl From for Span { + fn from(inner: fallback::Span) -> Self { + Span::Fallback(inner) + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Span::Compiler(s) => Debug::fmt(s, f), + Span::Fallback(s) => Debug::fmt(s, f), + } + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + match span { + Span::Compiler(s) => { + debug.field("span", &s); + } + Span::Fallback(s) => fallback::debug_span_field_if_nontrivial(debug, s), + } +} + +#[derive(Clone)] +pub(crate) enum Group { + Compiler(proc_macro::Group), + Fallback(fallback::Group), +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + match stream { + TokenStream::Compiler(tts) => { + let delimiter = match delimiter { + Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis, + Delimiter::Bracket => proc_macro::Delimiter::Bracket, + Delimiter::Brace => proc_macro::Delimiter::Brace, + Delimiter::None => proc_macro::Delimiter::None, + }; + Group::Compiler(proc_macro::Group::new(delimiter, tts.into_token_stream())) + } + TokenStream::Fallback(stream) => { + Group::Fallback(fallback::Group::new(delimiter, stream)) + } + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + match self { + Group::Compiler(g) => match g.delimiter() { + proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, + proc_macro::Delimiter::Bracket => Delimiter::Bracket, + proc_macro::Delimiter::Brace => Delimiter::Brace, + proc_macro::Delimiter::None => Delimiter::None, + }, + Group::Fallback(g) => g.delimiter(), + } + } + + pub(crate) fn stream(&self) -> TokenStream { + match self { + Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())), + Group::Fallback(g) => TokenStream::Fallback(g.stream()), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span()), + Group::Fallback(g) => Span::Fallback(g.span()), + } + } + + pub(crate) fn span_open(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_open()), + Group::Fallback(g) => Span::Fallback(g.span_open()), + } + } + + pub(crate) fn span_close(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_close()), + Group::Fallback(g) => Span::Fallback(g.span_close()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Group::Compiler(g), Span::Compiler(s)) => g.set_span(s), + (Group::Fallback(g), Span::Fallback(s)) => g.set_span(s), + (Group::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Group::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Group { + match self { + Group::Compiler(g) => g, + Group::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Group { + fn from(g: fallback::Group) -> Self { + Group::Fallback(g) + } +} + +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Display::fmt(group, formatter), + Group::Fallback(group) => Display::fmt(group, formatter), + } + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Debug::fmt(group, formatter), + Group::Fallback(group) => Debug::fmt(group, formatter), + } + } +} + +#[derive(Clone)] +pub(crate) enum Ident { + Compiler(proc_macro::Ident), + Fallback(fallback::Ident), +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_checked(string, s)), + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw_checked(string, s)), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Ident::Compiler(t) => Span::Compiler(t.span()), + Ident::Fallback(t) => Span::Fallback(t.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Ident::Compiler(t), Span::Compiler(s)) => t.set_span(s), + (Ident::Fallback(t), Span::Fallback(s)) => t.set_span(s), + (Ident::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Ident { + match self { + Ident::Compiler(s) => s, + Ident::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Ident { + fn from(inner: fallback::Ident) -> Self { + Ident::Fallback(inner) + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + match (self, other) { + (Ident::Compiler(t), Ident::Compiler(o)) => t.to_string() == o.to_string(), + (Ident::Fallback(t), Ident::Fallback(o)) => t == o, + (Ident::Compiler(_), Ident::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Ident::Compiler(_)) => mismatch(line!()), + } + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + match self { + Ident::Compiler(t) => t.to_string() == other, + Ident::Fallback(t) => t == other, + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Display::fmt(t, f), + Ident::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Debug::fmt(t, f), + Ident::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[derive(Clone)] +pub(crate) enum Literal { + Compiler(proc_macro::Literal), + Fallback(fallback::Literal), +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +macro_rules! unsuffixed_integers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +impl Literal { + pub(crate) fn from_str_checked(repr: &str) -> Result { + if inside_proc_macro() { + let literal = proc_macro::Literal::from_str_checked(repr)?; + Ok(Literal::Compiler(literal)) + } else { + let literal = fallback::Literal::from_str_checked(repr)?; + Ok(Literal::Fallback(literal)) + } + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::from_str_unchecked(repr)) + } else { + Literal::Fallback(unsafe { fallback::Literal::from_str_unchecked(repr) }) + } + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_integers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f32_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f32_unsuffixed(f)) + } + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f64_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f64_unsuffixed(f)) + } + } + + pub(crate) fn string(string: &str) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::string(string)) + } else { + Literal::Fallback(fallback::Literal::string(string)) + } + } + + pub(crate) fn character(ch: char) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::character(ch)) + } else { + Literal::Fallback(fallback::Literal::character(ch)) + } + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_byte_character))] + { + proc_macro::Literal::byte_character(byte) + } + + #[cfg(no_literal_byte_character)] + { + let fallback = fallback::Literal::byte_character(byte); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::byte_character(byte)) + } + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::byte_string(bytes)) + } else { + Literal::Fallback(fallback::Literal::byte_string(bytes)) + } + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_c_string))] + { + proc_macro::Literal::c_string(string) + } + + #[cfg(no_literal_c_string)] + { + let fallback = fallback::Literal::c_string(string); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::c_string(string)) + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Literal::Compiler(lit) => Span::Compiler(lit.span()), + Literal::Fallback(lit) => Span::Fallback(lit.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Literal::Compiler(lit), Span::Compiler(s)) => lit.set_span(s), + (Literal::Fallback(lit), Span::Fallback(s)) => lit.set_span(s), + (Literal::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Literal::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + match self { + #[cfg(proc_macro_span)] + Literal::Compiler(lit) => proc_macro_span::subspan(lit, range).map(Span::Compiler), + #[cfg(not(proc_macro_span))] + Literal::Compiler(_lit) => None, + Literal::Fallback(lit) => lit.subspan(range).map(Span::Fallback), + } + } + + fn unwrap_nightly(self) -> proc_macro::Literal { + match self { + Literal::Compiler(s) => s, + Literal::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Literal { + fn from(s: fallback::Literal) -> Self { + Literal::Fallback(s) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Display::fmt(t, f), + Literal::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Debug::fmt(t, f), + Literal::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + if inside_proc_macro() { + panic!( + "proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros" + ); + } else { + crate::fallback::invalidate_current_thread_spans(); + } +} diff --git a/rust/quote/README.md b/rust/quote/README.md new file mode 100644 index 000000000..186123eff --- /dev/null +++ b/rust/quote/README.md @@ -0,0 +1,12 @@ +# `quote` + +These source files come from the Rust `quote` crate, version 1.0.40 +(released 2025-03-12), hosted in the +repository, licensed under "Apache-2.0 OR MIT" and only modified to add +the SPDX license identifiers. + +For copyright details, please see: + + https://github.com/dtolnay/quote/blob/1.0.40/README.md#license + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-APACHE + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-MIT diff --git a/rust/quote/ext.rs b/rust/quote/ext.rs new file mode 100644 index 000000000..977d2f0c5 --- /dev/null +++ b/rust/quote/ext.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use super::ToTokens; +use core::iter; +use proc_macro2::{TokenStream, TokenTree}; + +/// TokenStream extension trait with methods for appending tokens. +/// +/// This trait is sealed and cannot be implemented outside of the `quote` crate. +pub trait TokenStreamExt: private::Sealed { + /// For use by `ToTokens` implementations. + /// + /// Appends the token specified to this list of tokens. + fn append(&mut self, token: U) + where + U: Into; + + /// For use by `ToTokens` implementations. + /// + /// ``` + /// # use quote::{quote, TokenStreamExt, ToTokens}; + /// # use proc_macro2::TokenStream; + /// # + /// struct X; + /// + /// impl ToTokens for X { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// tokens.append_all(&[true, false]); + /// } + /// } + /// + /// let tokens = quote!(#X); + /// assert_eq!(tokens.to_string(), "true false"); + /// ``` + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all of the items in the iterator `I`, separated by the tokens + /// `U`. + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all tokens in the iterator `I`, appending `U` after each + /// element, including after the last element of the iterator. + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; +} + +impl TokenStreamExt for TokenStream { + fn append(&mut self, token: U) + where + U: Into, + { + self.extend(iter::once(token.into())); + } + + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens, + { + for token in iter { + token.to_tokens(self); + } + } + + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for (i, token) in iter.into_iter().enumerate() { + if i > 0 { + op.to_tokens(self); + } + token.to_tokens(self); + } + } + + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for token in iter { + token.to_tokens(self); + term.to_tokens(self); + } + } +} + +mod private { + use proc_macro2::TokenStream; + + pub trait Sealed {} + + impl Sealed for TokenStream {} +} diff --git a/rust/quote/format.rs b/rust/quote/format.rs new file mode 100644 index 000000000..6e3d55b6e --- /dev/null +++ b/rust/quote/format.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Formatting macro for constructing `Ident`s. +/// +///
+/// +/// # Syntax +/// +/// Syntax is copied from the [`format!`] macro, supporting both positional and +/// named arguments. +/// +/// Only a limited set of formatting traits are supported. The current mapping +/// of format types to traits is: +/// +/// * `{}` ⇒ [`IdentFragment`] +/// * `{:o}` ⇒ [`Octal`](std::fmt::Octal) +/// * `{:x}` ⇒ [`LowerHex`](std::fmt::LowerHex) +/// * `{:X}` ⇒ [`UpperHex`](std::fmt::UpperHex) +/// * `{:b}` ⇒ [`Binary`](std::fmt::Binary) +/// +/// See [`std::fmt`] for more information. +/// +///
+/// +/// # IdentFragment +/// +/// Unlike `format!`, this macro uses the [`IdentFragment`] formatting trait by +/// default. This trait is like `Display`, with a few differences: +/// +/// * `IdentFragment` is only implemented for a limited set of types, such as +/// unsigned integers and strings. +/// * [`Ident`] arguments will have their `r#` prefixes stripped, if present. +/// +/// [`IdentFragment`]: crate::IdentFragment +/// [`Ident`]: proc_macro2::Ident +/// +///
+/// +/// # Hygiene +/// +/// The [`Span`] of the first `Ident` argument is used as the span of the final +/// identifier, falling back to [`Span::call_site`] when no identifiers are +/// provided. +/// +/// ``` +/// # use quote::format_ident; +/// # let ident = format_ident!("Ident"); +/// // If `ident` is an Ident, the span of `my_ident` will be inherited from it. +/// let my_ident = format_ident!("My{}{}", ident, "IsCool"); +/// assert_eq!(my_ident, "MyIdentIsCool"); +/// ``` +/// +/// Alternatively, the span can be overridden by passing the `span` named +/// argument. +/// +/// ``` +/// # use quote::format_ident; +/// # const IGNORE_TOKENS: &'static str = stringify! { +/// let my_span = /* ... */; +/// # }; +/// # let my_span = proc_macro2::Span::call_site(); +/// format_ident!("MyIdent", span = my_span); +/// ``` +/// +/// [`Span`]: proc_macro2::Span +/// [`Span::call_site`]: proc_macro2::Span::call_site +/// +///


+/// +/// # Panics +/// +/// This method will panic if the resulting formatted string is not a valid +/// identifier. +/// +///
+/// +/// # Examples +/// +/// Composing raw and non-raw identifiers: +/// ``` +/// # use quote::format_ident; +/// let my_ident = format_ident!("My{}", "Ident"); +/// assert_eq!(my_ident, "MyIdent"); +/// +/// let raw = format_ident!("r#Raw"); +/// assert_eq!(raw, "r#Raw"); +/// +/// let my_ident_raw = format_ident!("{}Is{}", my_ident, raw); +/// assert_eq!(my_ident_raw, "MyIdentIsRaw"); +/// ``` +/// +/// Integer formatting options: +/// ``` +/// # use quote::format_ident; +/// let num: u32 = 10; +/// +/// let decimal = format_ident!("Id_{}", num); +/// assert_eq!(decimal, "Id_10"); +/// +/// let octal = format_ident!("Id_{:o}", num); +/// assert_eq!(octal, "Id_12"); +/// +/// let binary = format_ident!("Id_{:b}", num); +/// assert_eq!(binary, "Id_1010"); +/// +/// let lower_hex = format_ident!("Id_{:x}", num); +/// assert_eq!(lower_hex, "Id_a"); +/// +/// let upper_hex = format_ident!("Id_{:X}", num); +/// assert_eq!(upper_hex, "Id_A"); +/// ``` +#[macro_export] +macro_rules! format_ident { + ($fmt:expr) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ]) + }; + + ($fmt:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ] $($rest)*) + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! format_ident_impl { + // Final state + ([$span:expr, $($fmt:tt)*]) => { + $crate::__private::mk_ident( + &$crate::__private::format!($($fmt)*), + $span, + ) + }; + + // Span argument + ([$old:expr, $($fmt:tt)*] span = $span:expr) => { + $crate::format_ident_impl!([$old, $($fmt)*] span = $span,) + }; + ([$old:expr, $($fmt:tt)*] span = $span:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::Some::<$crate::__private::Span>($span), + $($fmt)* + ] $($rest)*) + }; + + // Named argument + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $name = $arg,) + }; + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, $name = arg] $($rest)*), + } + }; + + // Positional argument + ([$span:expr, $($fmt:tt)*] $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $arg,) + }; + ([$span:expr, $($fmt:tt)*] $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, arg] $($rest)*), + } + }; +} diff --git a/rust/quote/ident_fragment.rs b/rust/quote/ident_fragment.rs new file mode 100644 index 000000000..d98106f17 --- /dev/null +++ b/rust/quote/ident_fragment.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::borrow::Cow; +use core::fmt; +use proc_macro2::{Ident, Span}; + +/// Specialized formatting trait used by `format_ident!`. +/// +/// [`Ident`] arguments formatted using this trait will have their `r#` prefix +/// stripped, if present. +/// +/// See [`format_ident!`] for more information. +/// +/// [`format_ident!`]: crate::format_ident +pub trait IdentFragment { + /// Format this value as an identifier fragment. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result; + + /// Span associated with this `IdentFragment`. + /// + /// If non-`None`, may be inherited by formatted identifiers. + fn span(&self) -> Option { + None + } +} + +impl IdentFragment for &T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for &mut T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for Ident { + fn span(&self) -> Option { + Some(self.span()) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let id = self.to_string(); + if let Some(id) = id.strip_prefix("r#") { + fmt::Display::fmt(id, f) + } else { + fmt::Display::fmt(&id[..], f) + } + } +} + +impl IdentFragment for Cow<'_, T> +where + T: IdentFragment + ToOwned + ?Sized, +{ + fn span(&self) -> Option { + T::span(self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + T::fmt(self, f) + } +} + +// Limited set of types which this is implemented for, as we want to avoid types +// which will often include non-identifier characters in their `Display` impl. +macro_rules! ident_fragment_display { + ($($T:ty),*) => { + $( + impl IdentFragment for $T { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self, f) + } + } + )* + }; +} + +ident_fragment_display!(bool, str, String, char); +ident_fragment_display!(u8, u16, u32, u64, u128, usize); diff --git a/rust/quote/lib.rs b/rust/quote/lib.rs new file mode 100644 index 000000000..cc1637660 --- /dev/null +++ b/rust/quote/lib.rs @@ -0,0 +1,1456 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! [![github]](https://github.com/dtolnay/quote) [![crates-io]](https://crates.io/crates/quote) [![docs-rs]](https://docs.rs/quote) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! This crate provides the [`quote!`] macro for turning Rust syntax tree data +//! structures into tokens of source code. +//! +//! Procedural macros in Rust receive a stream of tokens as input, execute +//! arbitrary Rust code to determine how to manipulate those tokens, and produce +//! a stream of tokens to hand back to the compiler to compile into the caller's +//! crate. Quasi-quoting is a solution to one piece of that — producing +//! tokens to return to the compiler. +//! +//! The idea of quasi-quoting is that we write *code* that we treat as *data*. +//! Within the `quote!` macro, we can write what looks like code to our text +//! editor or IDE. We get all the benefits of the editor's brace matching, +//! syntax highlighting, indentation, and maybe autocompletion. But rather than +//! compiling that as code into the current crate, we can treat it as data, pass +//! it around, mutate it, and eventually hand it back to the compiler as tokens +//! to compile into the macro caller's crate. +//! +//! This crate is motivated by the procedural macro use case, but is a +//! general-purpose Rust quasi-quoting library and is not specific to procedural +//! macros. +//! +//! ```toml +//! [dependencies] +//! quote = "1.0" +//! ``` +//! +//!
+//! +//! # Example +//! +//! The following quasi-quoted block of code is something you might find in [a] +//! procedural macro having to do with data structure serialization. The `#var` +//! syntax performs interpolation of runtime variables into the quoted tokens. +//! Check out the documentation of the [`quote!`] macro for more detail about +//! the syntax. See also the [`quote_spanned!`] macro which is important for +//! implementing hygienic procedural macros. +//! +//! [a]: https://serde.rs/ +//! +//! ``` +//! # use quote::quote; +//! # +//! # let generics = ""; +//! # let where_clause = ""; +//! # let field_ty = ""; +//! # let item_ty = ""; +//! # let path = ""; +//! # let value = ""; +//! # +//! let tokens = quote! { +//! struct SerializeWith #generics #where_clause { +//! value: &'a #field_ty, +//! phantom: core::marker::PhantomData<#item_ty>, +//! } +//! +//! impl #generics serde::Serialize for SerializeWith #generics #where_clause { +//! fn serialize(&self, serializer: S) -> Result +//! where +//! S: serde::Serializer, +//! { +//! #path(self.value, serializer) +//! } +//! } +//! +//! SerializeWith { +//! value: #value, +//! phantom: core::marker::PhantomData::<#item_ty>, +//! } +//! }; +//! ``` +//! +//!
+//! +//! # Non-macro code generators +//! +//! When using `quote` in a build.rs or main.rs and writing the output out to a +//! file, consider having the code generator pass the tokens through +//! [prettyplease] before writing. This way if an error occurs in the generated +//! code it is convenient for a human to read and debug. +//! +//! [prettyplease]: https://github.com/dtolnay/prettyplease + +// Quote types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/quote/1.0.40")] +#![allow( + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::missing_errors_doc, + clippy::missing_panics_doc, + clippy::module_name_repetitions, + clippy::needless_lifetimes, + // false positive https://github.com/rust-lang/rust-clippy/issues/6983 + clippy::wrong_self_convention, +)] + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod ext; +mod format; +mod ident_fragment; +mod to_tokens; + +// Not public API. +#[doc(hidden)] +#[path = "runtime.rs"] +pub mod __private; + +pub use crate::ext::TokenStreamExt; +pub use crate::ident_fragment::IdentFragment; +pub use crate::to_tokens::ToTokens; + +// Not public API. +#[doc(hidden)] +pub mod spanned; + +macro_rules! __quote { + ($quote:item) => { + /// The whole point. + /// + /// Performs variable interpolation against the input and produces it as + /// [`proc_macro2::TokenStream`]. + /// + /// Note: for returning tokens to the compiler in a procedural macro, use + /// `.into()` on the result to convert to [`proc_macro::TokenStream`]. + /// + ///
+ /// + /// # Interpolation + /// + /// Variable interpolation is done with `#var` (similar to `$var` in + /// `macro_rules!` macros). This grabs the `var` variable that is currently in + /// scope and inserts it in that location in the output tokens. Any type + /// implementing the [`ToTokens`] trait can be interpolated. This includes most + /// Rust primitive types as well as most of the syntax tree types from the [Syn] + /// crate. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// Repetition is done using `#(...)*` or `#(...),*` again similar to + /// `macro_rules!`. This iterates through the elements of any variable + /// interpolated within the repetition and inserts a copy of the repetition body + /// for each one. The variables in an interpolation may be a `Vec`, slice, + /// `BTreeSet`, or any `Iterator`. + /// + /// - `#(#var)*` — no separators + /// - `#(#var),*` — the character before the asterisk is used as a separator + /// - `#( struct #var; )*` — the repetition can contain other tokens + /// - `#( #k => println!("{}", #v), )*` — even multiple interpolations + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote!` + /// invocation are spanned with [`Span::call_site()`]. + /// + /// [`Span::call_site()`]: proc_macro2::Span::call_site + /// + /// A different span can be provided through the [`quote_spanned!`] macro. + /// + ///
+ /// + /// # Return type + /// + /// The macro evaluates to an expression of type `proc_macro2::TokenStream`. + /// Meanwhile Rust procedural macros are expected to return the type + /// `proc_macro::TokenStream`. + /// + /// The difference between the two types is that `proc_macro` types are entirely + /// specific to procedural macros and cannot ever exist in code outside of a + /// procedural macro, while `proc_macro2` types may exist anywhere including + /// tests and non-macro code like main.rs and build.rs. This is why even the + /// procedural macro ecosystem is largely built around `proc_macro2`, because + /// that ensures the libraries are unit testable and accessible in non-macro + /// contexts. + /// + /// There is a [`From`]-conversion in both directions so returning the output of + /// `quote!` from a procedural macro usually looks like `tokens.into()` or + /// `proc_macro::TokenStream::from(tokens)`. + /// + ///
+ /// + /// # Examples + /// + /// ### Procedural macro + /// + /// The structure of a basic procedural macro is as follows. Refer to the [Syn] + /// crate for further useful guidance on using `quote!` as part of a procedural + /// macro. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// ``` + /// # #[cfg(any())] + /// extern crate proc_macro; + /// # extern crate proc_macro2; + /// + /// # #[cfg(any())] + /// use proc_macro::TokenStream; + /// # use proc_macro2::TokenStream; + /// use quote::quote; + /// + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// #[proc_macro_derive(HeapSize)] + /// # }; + /// pub fn derive_heap_size(input: TokenStream) -> TokenStream { + /// // Parse the input and figure out what implementation to generate... + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let name = /* ... */; + /// let expr = /* ... */; + /// # }; + /// # + /// # let name = 0; + /// # let expr = 0; + /// + /// let expanded = quote! { + /// // The generated impl. + /// impl heapsize::HeapSize for #name { + /// fn heap_size_of_children(&self) -> usize { + /// #expr + /// } + /// } + /// }; + /// + /// // Hand the output tokens back to the compiler. + /// TokenStream::from(expanded) + /// } + /// ``` + /// + ///


+ /// + /// ### Combining quoted fragments + /// + /// Usually you don't end up constructing an entire final `TokenStream` in one + /// piece. Different parts may come from different helper functions. The tokens + /// produced by `quote!` themselves implement `ToTokens` and so can be + /// interpolated into later `quote!` invocations to build up a final result. + /// + /// ``` + /// # use quote::quote; + /// # + /// let type_definition = quote! {...}; + /// let methods = quote! {...}; + /// + /// let tokens = quote! { + /// #type_definition + /// #methods + /// }; + /// ``` + /// + ///


+ /// + /// ### Constructing identifiers + /// + /// Suppose we have an identifier `ident` which came from somewhere in a macro + /// input and we need to modify it in some way for the macro output. Let's + /// consider prepending the identifier with an underscore. + /// + /// Simply interpolating the identifier next to an underscore will not have the + /// behavior of concatenating them. The underscore and the identifier will + /// continue to be two separate tokens as if you had written `_ x`. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// // incorrect + /// quote! { + /// let mut _#ident = 0; + /// } + /// # ; + /// ``` + /// + /// The solution is to build a new identifier token with the correct value. As + /// this is such a common case, the [`format_ident!`] macro provides a + /// convenient utility for doing so correctly. + /// + /// ``` + /// # use proc_macro2::{Ident, Span}; + /// # use quote::{format_ident, quote}; + /// # + /// # let ident = Ident::new("i", Span::call_site()); + /// # + /// let varname = format_ident!("_{}", ident); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + /// Alternatively, the APIs provided by Syn and proc-macro2 can be used to + /// directly build the identifier. This is roughly equivalent to the above, but + /// will not handle `ident` being a raw identifier. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// let concatenated = format!("_{}", ident); + /// let varname = syn::Ident::new(&concatenated, ident.span()); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Making method calls + /// + /// Let's say our macro requires some type specified in the macro input to have + /// a constructor called `new`. We have the type in a variable called + /// `field_type` of type `syn::Type` and want to invoke the constructor. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// // incorrect + /// quote! { + /// let value = #field_type::new(); + /// } + /// # ; + /// ``` + /// + /// This works only sometimes. If `field_type` is `String`, the expanded code + /// contains `String::new()` which is fine. But if `field_type` is something + /// like `Vec` then the expanded code is `Vec::new()` which is invalid + /// syntax. Ordinarily in handwritten Rust we would write `Vec::::new()` + /// but for macros often the following is more convenient. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type>::new(); + /// } + /// # ; + /// ``` + /// + /// This expands to `>::new()` which behaves correctly. + /// + /// A similar pattern is appropriate for trait methods. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type as core::default::Default>::default(); + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Interpolating text inside of doc comments + /// + /// Neither doc comments nor string literals get interpolation behavior in + /// quote: + /// + /// ```compile_fail + /// quote! { + /// /// try to interpolate: #ident + /// /// + /// /// ... + /// } + /// ``` + /// + /// ```compile_fail + /// quote! { + /// #[doc = "try to interpolate: #ident"] + /// } + /// ``` + /// + /// Instead the best way to build doc comments that involve variables is by + /// formatting the doc string literal outside of quote. + /// + /// ```rust + /// # use proc_macro2::{Ident, Span}; + /// # use quote::quote; + /// # + /// # const IGNORE: &str = stringify! { + /// let msg = format!(...); + /// # }; + /// # + /// # let ident = Ident::new("var", Span::call_site()); + /// # let msg = format!("try to interpolate: {}", ident); + /// quote! { + /// #[doc = #msg] + /// /// + /// /// ... + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Indexing into a tuple struct + /// + /// When interpolating indices of a tuple or tuple struct, we need them not to + /// appears suffixed as integer literals by interpolating them as [`syn::Index`] + /// instead. + /// + /// [`syn::Index`]: https://docs.rs/syn/2.0/syn/struct.Index.html + /// + /// ```compile_fail + /// let i = 0usize..self.fields.len(); + /// + /// // expands to 0 + self.0usize.heap_size() + self.1usize.heap_size() + ... + /// // which is not valid syntax + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// ``` + /// + /// ``` + /// # use proc_macro2::{Ident, TokenStream}; + /// # use quote::quote; + /// # + /// # mod syn { + /// # use proc_macro2::{Literal, TokenStream}; + /// # use quote::{ToTokens, TokenStreamExt}; + /// # + /// # pub struct Index(usize); + /// # + /// # impl From for Index { + /// # fn from(i: usize) -> Self { + /// # Index(i) + /// # } + /// # } + /// # + /// # impl ToTokens for Index { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # tokens.append(Literal::usize_unsuffixed(self.0)); + /// # } + /// # } + /// # } + /// # + /// # struct Struct { + /// # fields: Vec, + /// # } + /// # + /// # impl Struct { + /// # fn example(&self) -> TokenStream { + /// let i = (0..self.fields.len()).map(syn::Index::from); + /// + /// // expands to 0 + self.0.heap_size() + self.1.heap_size() + ... + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// # } + /// # } + /// ``` + $quote + }; +} + +#[cfg(doc)] +__quote![ + #[macro_export] + macro_rules! quote { + ($($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote![ + #[macro_export] + macro_rules! quote { + () => { + $crate::__private::TokenStream::new() + }; + + // Special case rule for a single tt, for performance. + ($tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt _s} + _s + }}; + + // Special case rules for two tts, for performance. + (# $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt1 _s} + $crate::quote_token!{$tt2 _s} + _s + }}; + + // Rule for any other number of tokens. + ($($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_each_token!{_s $($tt)*} + _s + }}; + } +]; + +macro_rules! __quote_spanned { + ($quote_spanned:item) => { + /// Same as `quote!`, but applies a given span to all tokens originating within + /// the macro invocation. + /// + ///
+ /// + /// # Syntax + /// + /// A span expression of type [`Span`], followed by `=>`, followed by the tokens + /// to quote. The span expression should be brief — use a variable for + /// anything more than a few characters. There should be no space before the + /// `=>` token. + /// + /// [`Span`]: proc_macro2::Span + /// + /// ``` + /// # use proc_macro2::Span; + /// # use quote::quote_spanned; + /// # + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let span = /* ... */; + /// # }; + /// # let span = Span::call_site(); + /// # let init = 0; + /// + /// // On one line, use parentheses. + /// let tokens = quote_spanned!(span=> Box::into_raw(Box::new(#init))); + /// + /// // On multiple lines, place the span at the top and use braces. + /// let tokens = quote_spanned! {span=> + /// Box::into_raw(Box::new(#init)) + /// }; + /// ``` + /// + /// The lack of space before the `=>` should look jarring to Rust programmers + /// and this is intentional. The formatting is designed to be visibly + /// off-balance and draw the eye a particular way, due to the span expression + /// being evaluated in the context of the procedural macro and the remaining + /// tokens being evaluated in the generated code. + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote_spanned!` + /// invocation are spanned with the given span argument. + /// + ///
+ /// + /// # Example + /// + /// The following procedural macro code uses `quote_spanned!` to assert that a + /// particular Rust type implements the [`Sync`] trait so that references can be + /// safely shared between threads. + /// + /// ``` + /// # use quote::{quote_spanned, TokenStreamExt, ToTokens}; + /// # use proc_macro2::{Span, TokenStream}; + /// # + /// # struct Type; + /// # + /// # impl Type { + /// # fn span(&self) -> Span { + /// # Span::call_site() + /// # } + /// # } + /// # + /// # impl ToTokens for Type { + /// # fn to_tokens(&self, _tokens: &mut TokenStream) {} + /// # } + /// # + /// # let ty = Type; + /// # let call_site = Span::call_site(); + /// # + /// let ty_span = ty.span(); + /// let assert_sync = quote_spanned! {ty_span=> + /// struct _AssertSync where #ty: Sync; + /// }; + /// ``` + /// + /// If the assertion fails, the user will see an error like the following. The + /// input span of their type is highlighted in the error. + /// + /// ```text + /// error[E0277]: the trait bound `*const (): std::marker::Sync` is not satisfied + /// --> src/main.rs:10:21 + /// | + /// 10 | static ref PTR: *const () = &(); + /// | ^^^^^^^^^ `*const ()` cannot be shared between threads safely + /// ``` + /// + /// In this example it is important for the where-clause to be spanned with the + /// line/column information of the user's input type so that error messages are + /// placed appropriately by the compiler. + $quote_spanned + }; +} + +#[cfg(doc)] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=> $($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=>) => {{ + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::__private::TokenStream::new() + }}; + + // Special case rule for a single tt, for performance. + ($span:expr=> $tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt _s _span} + _s + }}; + + // Special case rules for two tts, for performance. + ($span:expr=> # $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($span:expr=> $tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt1 _s _span} + $crate::quote_token_spanned!{$tt2 _s _span} + _s + }}; + + // Rule for any other number of tokens. + ($span:expr=> $($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_each_token_spanned!{_s _span $($tt)*} + _s + }}; + } +]; + +// Extract the names of all #metavariables and pass them to the $call macro. +// +// in: pounded_var_names!(then!(...) a #b c #( #d )* #e) +// out: then!(... b); +// then!(... d); +// then!(... e); +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names { + ($call:ident! $extra:tt $($tts:tt)*) => { + $crate::pounded_var_names_with_context!{$call! $extra + (@ $($tts)*) + ($($tts)* @) + } + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names_with_context { + ($call:ident! $extra:tt ($($b1:tt)*) ($($curr:tt)*)) => { + $( + $crate::pounded_var_with_context!{$call! $extra $b1 $curr} + )* + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_with_context { + ($call:ident! $extra:tt $b1:tt ( $($inner:tt)* )) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt [ $($inner:tt)* ]) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt { $($inner:tt)* }) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident!($($extra:tt)*) # $var:ident) => { + $crate::$call!($($extra)* $var); + }; + + ($call:ident! $extra:tt $b1:tt $curr:tt) => {}; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_into_iter { + ($has_iter:ident $var:ident) => { + // `mut` may be unused if $var occurs multiple times in the list. + #[allow(unused_mut)] + let (mut $var, i) = $var.quote_into_iter(); + let $has_iter = $has_iter | i; + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_next_or_break { + ($var:ident) => { + let $var = match $var.next() { + Some(_x) => $crate::__private::RepInterp(_x), + None => break, + }; + }; +} + +// The obvious way to write this macro is as a tt muncher. This implementation +// does something more complex for two reasons. +// +// - With a tt muncher it's easy to hit Rust's built-in recursion_limit, which +// this implementation avoids because it isn't tail recursive. +// +// - Compile times for a tt muncher are quadratic relative to the length of +// the input. This implementation is linear, so it will be faster +// (potentially much faster) for big inputs. However, the constant factors +// of this implementation are higher than that of a tt muncher, so it is +// somewhat slower than a tt muncher if there are many invocations with +// short inputs. +// +// An invocation like this: +// +// quote_each_token!(_s a b c d e f g h i j); +// +// expands to this: +// +// quote_tokens_with_context!(_s +// (@ @ @ @ @ @ a b c d e f g h i j) +// (@ @ @ @ @ a b c d e f g h i j @) +// (@ @ @ @ a b c d e f g h i j @ @) +// (@ @ @ (a) (b) (c) (d) (e) (f) (g) (h) (i) (j) @ @ @) +// (@ @ a b c d e f g h i j @ @ @ @) +// (@ a b c d e f g h i j @ @ @ @ @) +// (a b c d e f g h i j @ @ @ @ @ @) +// ); +// +// which gets transposed and expanded to this: +// +// quote_token_with_context!(_s @ @ @ @ @ @ a); +// quote_token_with_context!(_s @ @ @ @ @ a b); +// quote_token_with_context!(_s @ @ @ @ a b c); +// quote_token_with_context!(_s @ @ @ (a) b c d); +// quote_token_with_context!(_s @ @ a (b) c d e); +// quote_token_with_context!(_s @ a b (c) d e f); +// quote_token_with_context!(_s a b c (d) e f g); +// quote_token_with_context!(_s b c d (e) f g h); +// quote_token_with_context!(_s c d e (f) g h i); +// quote_token_with_context!(_s d e f (g) h i j); +// quote_token_with_context!(_s e f g (h) i j @); +// quote_token_with_context!(_s f g h (i) j @ @); +// quote_token_with_context!(_s g h i (j) @ @ @); +// quote_token_with_context!(_s h i j @ @ @ @); +// quote_token_with_context!(_s i j @ @ @ @ @); +// quote_token_with_context!(_s j @ @ @ @ @ @); +// +// Without having used muncher-style recursion, we get one invocation of +// quote_token_with_context for each original tt, with three tts of context on +// either side. This is enough for the longest possible interpolation form (a +// repetition with separator, as in `# (#var) , *`) to be fully represented with +// the first or last tt in the middle. +// +// The middle tt (surrounded by parentheses) is the tt being processed. +// +// - When it is a `#`, quote_token_with_context can do an interpolation. The +// interpolation kind will depend on the three subsequent tts. +// +// - When it is within a later part of an interpolation, it can be ignored +// because the interpolation has already been done. +// +// - When it is not part of an interpolation it can be pushed as a single +// token into the output. +// +// - When the middle token is an unparenthesized `@`, that call is one of the +// first 3 or last 3 calls of quote_token_with_context and does not +// correspond to one of the original input tokens, so turns into nothing. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token { + ($tokens:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context!{$tokens + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token_spanned { + ($tokens:ident $span:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context_spanned!{$tokens $span + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context { + ($tokens:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context!{$tokens $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context_spanned { + ($tokens:ident $span:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context_spanned!{$tokens $span $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context { + // Unparenthesized `@` indicates this call does not correspond to one of the + // original input tokens. Ignore it. + ($tokens:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with no separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + // This is `while true` instead of `loop` because if there are no + // iterators used inside of this repetition then the body would not + // contain any `break`, so the compiler would emit unreachable code + // warnings on anything below the loop. We use has_iter to detect and + // fail to compile when there are no iterators, so here we just work + // around the unneeded extra warning. + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token!{$sep $tokens} + } + _i += 1; + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + // (A special case for `#(var)**`, where the first `*` is treated as the + // repetition symbol and the second `*` is treated as an ordinary token.) + ($tokens:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token!{* $tokens} + }; + // ... and one step later. + ($tokens:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A non-repetition interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + // An ordinary token, not part of any interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token!{$curr $tokens} + }; +} + +// See the explanation on quote_each_token, and on the individual rules of +// quote_token_with_context. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context_spanned { + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token_spanned!{$sep $tokens $span} + } + _i += 1; + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token_spanned!{* $tokens $span} + }; + ($tokens:ident $span:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + ($tokens:ident $span:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token_spanned!{$curr $tokens $span} + }; +} + +// These rules are ordered by approximate token frequency, at least for the +// first 10 or so, to improve compile times. Having `ident` first is by far the +// most important because it's typically 2-3x more common than the next most +// common token. +// +// Separately, we put the token being matched in the very front so that failing +// rules may fail to match as quickly as possible. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token { + ($ident:ident $tokens:ident) => { + $crate::__private::push_ident(&mut $tokens, stringify!($ident)); + }; + + (:: $tokens:ident) => { + $crate::__private::push_colon2(&mut $tokens); + }; + + (( $($inner:tt)* ) $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Parenthesis, + $crate::quote!($($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Bracket, + $crate::quote!($($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Brace, + $crate::quote!($($inner)*), + ); + }; + + (# $tokens:ident) => { + $crate::__private::push_pound(&mut $tokens); + }; + + (, $tokens:ident) => { + $crate::__private::push_comma(&mut $tokens); + }; + + (. $tokens:ident) => { + $crate::__private::push_dot(&mut $tokens); + }; + + (; $tokens:ident) => { + $crate::__private::push_semi(&mut $tokens); + }; + + (: $tokens:ident) => { + $crate::__private::push_colon(&mut $tokens); + }; + + (+ $tokens:ident) => { + $crate::__private::push_add(&mut $tokens); + }; + + (+= $tokens:ident) => { + $crate::__private::push_add_eq(&mut $tokens); + }; + + (& $tokens:ident) => { + $crate::__private::push_and(&mut $tokens); + }; + + (&& $tokens:ident) => { + $crate::__private::push_and_and(&mut $tokens); + }; + + (&= $tokens:ident) => { + $crate::__private::push_and_eq(&mut $tokens); + }; + + (@ $tokens:ident) => { + $crate::__private::push_at(&mut $tokens); + }; + + (! $tokens:ident) => { + $crate::__private::push_bang(&mut $tokens); + }; + + (^ $tokens:ident) => { + $crate::__private::push_caret(&mut $tokens); + }; + + (^= $tokens:ident) => { + $crate::__private::push_caret_eq(&mut $tokens); + }; + + (/ $tokens:ident) => { + $crate::__private::push_div(&mut $tokens); + }; + + (/= $tokens:ident) => { + $crate::__private::push_div_eq(&mut $tokens); + }; + + (.. $tokens:ident) => { + $crate::__private::push_dot2(&mut $tokens); + }; + + (... $tokens:ident) => { + $crate::__private::push_dot3(&mut $tokens); + }; + + (..= $tokens:ident) => { + $crate::__private::push_dot_dot_eq(&mut $tokens); + }; + + (= $tokens:ident) => { + $crate::__private::push_eq(&mut $tokens); + }; + + (== $tokens:ident) => { + $crate::__private::push_eq_eq(&mut $tokens); + }; + + (>= $tokens:ident) => { + $crate::__private::push_ge(&mut $tokens); + }; + + (> $tokens:ident) => { + $crate::__private::push_gt(&mut $tokens); + }; + + (<= $tokens:ident) => { + $crate::__private::push_le(&mut $tokens); + }; + + (< $tokens:ident) => { + $crate::__private::push_lt(&mut $tokens); + }; + + (*= $tokens:ident) => { + $crate::__private::push_mul_eq(&mut $tokens); + }; + + (!= $tokens:ident) => { + $crate::__private::push_ne(&mut $tokens); + }; + + (| $tokens:ident) => { + $crate::__private::push_or(&mut $tokens); + }; + + (|= $tokens:ident) => { + $crate::__private::push_or_eq(&mut $tokens); + }; + + (|| $tokens:ident) => { + $crate::__private::push_or_or(&mut $tokens); + }; + + (? $tokens:ident) => { + $crate::__private::push_question(&mut $tokens); + }; + + (-> $tokens:ident) => { + $crate::__private::push_rarrow(&mut $tokens); + }; + + (<- $tokens:ident) => { + $crate::__private::push_larrow(&mut $tokens); + }; + + (% $tokens:ident) => { + $crate::__private::push_rem(&mut $tokens); + }; + + (%= $tokens:ident) => { + $crate::__private::push_rem_eq(&mut $tokens); + }; + + (=> $tokens:ident) => { + $crate::__private::push_fat_arrow(&mut $tokens); + }; + + (<< $tokens:ident) => { + $crate::__private::push_shl(&mut $tokens); + }; + + (<<= $tokens:ident) => { + $crate::__private::push_shl_eq(&mut $tokens); + }; + + (>> $tokens:ident) => { + $crate::__private::push_shr(&mut $tokens); + }; + + (>>= $tokens:ident) => { + $crate::__private::push_shr_eq(&mut $tokens); + }; + + (* $tokens:ident) => { + $crate::__private::push_star(&mut $tokens); + }; + + (- $tokens:ident) => { + $crate::__private::push_sub(&mut $tokens); + }; + + (-= $tokens:ident) => { + $crate::__private::push_sub_eq(&mut $tokens); + }; + + ($lifetime:lifetime $tokens:ident) => { + $crate::__private::push_lifetime(&mut $tokens, stringify!($lifetime)); + }; + + (_ $tokens:ident) => { + $crate::__private::push_underscore(&mut $tokens); + }; + + ($other:tt $tokens:ident) => { + $crate::__private::parse(&mut $tokens, stringify!($other)); + }; +} + +// See the comment above `quote_token!` about the rule ordering. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_spanned { + ($ident:ident $tokens:ident $span:ident) => { + $crate::__private::push_ident_spanned(&mut $tokens, $span, stringify!($ident)); + }; + + (:: $tokens:ident $span:ident) => { + $crate::__private::push_colon2_spanned(&mut $tokens, $span); + }; + + (( $($inner:tt)* ) $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Parenthesis, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Bracket, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Brace, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + (# $tokens:ident $span:ident) => { + $crate::__private::push_pound_spanned(&mut $tokens, $span); + }; + + (, $tokens:ident $span:ident) => { + $crate::__private::push_comma_spanned(&mut $tokens, $span); + }; + + (. $tokens:ident $span:ident) => { + $crate::__private::push_dot_spanned(&mut $tokens, $span); + }; + + (; $tokens:ident $span:ident) => { + $crate::__private::push_semi_spanned(&mut $tokens, $span); + }; + + (: $tokens:ident $span:ident) => { + $crate::__private::push_colon_spanned(&mut $tokens, $span); + }; + + (+ $tokens:ident $span:ident) => { + $crate::__private::push_add_spanned(&mut $tokens, $span); + }; + + (+= $tokens:ident $span:ident) => { + $crate::__private::push_add_eq_spanned(&mut $tokens, $span); + }; + + (& $tokens:ident $span:ident) => { + $crate::__private::push_and_spanned(&mut $tokens, $span); + }; + + (&& $tokens:ident $span:ident) => { + $crate::__private::push_and_and_spanned(&mut $tokens, $span); + }; + + (&= $tokens:ident $span:ident) => { + $crate::__private::push_and_eq_spanned(&mut $tokens, $span); + }; + + (@ $tokens:ident $span:ident) => { + $crate::__private::push_at_spanned(&mut $tokens, $span); + }; + + (! $tokens:ident $span:ident) => { + $crate::__private::push_bang_spanned(&mut $tokens, $span); + }; + + (^ $tokens:ident $span:ident) => { + $crate::__private::push_caret_spanned(&mut $tokens, $span); + }; + + (^= $tokens:ident $span:ident) => { + $crate::__private::push_caret_eq_spanned(&mut $tokens, $span); + }; + + (/ $tokens:ident $span:ident) => { + $crate::__private::push_div_spanned(&mut $tokens, $span); + }; + + (/= $tokens:ident $span:ident) => { + $crate::__private::push_div_eq_spanned(&mut $tokens, $span); + }; + + (.. $tokens:ident $span:ident) => { + $crate::__private::push_dot2_spanned(&mut $tokens, $span); + }; + + (... $tokens:ident $span:ident) => { + $crate::__private::push_dot3_spanned(&mut $tokens, $span); + }; + + (..= $tokens:ident $span:ident) => { + $crate::__private::push_dot_dot_eq_spanned(&mut $tokens, $span); + }; + + (= $tokens:ident $span:ident) => { + $crate::__private::push_eq_spanned(&mut $tokens, $span); + }; + + (== $tokens:ident $span:ident) => { + $crate::__private::push_eq_eq_spanned(&mut $tokens, $span); + }; + + (>= $tokens:ident $span:ident) => { + $crate::__private::push_ge_spanned(&mut $tokens, $span); + }; + + (> $tokens:ident $span:ident) => { + $crate::__private::push_gt_spanned(&mut $tokens, $span); + }; + + (<= $tokens:ident $span:ident) => { + $crate::__private::push_le_spanned(&mut $tokens, $span); + }; + + (< $tokens:ident $span:ident) => { + $crate::__private::push_lt_spanned(&mut $tokens, $span); + }; + + (*= $tokens:ident $span:ident) => { + $crate::__private::push_mul_eq_spanned(&mut $tokens, $span); + }; + + (!= $tokens:ident $span:ident) => { + $crate::__private::push_ne_spanned(&mut $tokens, $span); + }; + + (| $tokens:ident $span:ident) => { + $crate::__private::push_or_spanned(&mut $tokens, $span); + }; + + (|= $tokens:ident $span:ident) => { + $crate::__private::push_or_eq_spanned(&mut $tokens, $span); + }; + + (|| $tokens:ident $span:ident) => { + $crate::__private::push_or_or_spanned(&mut $tokens, $span); + }; + + (? $tokens:ident $span:ident) => { + $crate::__private::push_question_spanned(&mut $tokens, $span); + }; + + (-> $tokens:ident $span:ident) => { + $crate::__private::push_rarrow_spanned(&mut $tokens, $span); + }; + + (<- $tokens:ident $span:ident) => { + $crate::__private::push_larrow_spanned(&mut $tokens, $span); + }; + + (% $tokens:ident $span:ident) => { + $crate::__private::push_rem_spanned(&mut $tokens, $span); + }; + + (%= $tokens:ident $span:ident) => { + $crate::__private::push_rem_eq_spanned(&mut $tokens, $span); + }; + + (=> $tokens:ident $span:ident) => { + $crate::__private::push_fat_arrow_spanned(&mut $tokens, $span); + }; + + (<< $tokens:ident $span:ident) => { + $crate::__private::push_shl_spanned(&mut $tokens, $span); + }; + + (<<= $tokens:ident $span:ident) => { + $crate::__private::push_shl_eq_spanned(&mut $tokens, $span); + }; + + (>> $tokens:ident $span:ident) => { + $crate::__private::push_shr_spanned(&mut $tokens, $span); + }; + + (>>= $tokens:ident $span:ident) => { + $crate::__private::push_shr_eq_spanned(&mut $tokens, $span); + }; + + (* $tokens:ident $span:ident) => { + $crate::__private::push_star_spanned(&mut $tokens, $span); + }; + + (- $tokens:ident $span:ident) => { + $crate::__private::push_sub_spanned(&mut $tokens, $span); + }; + + (-= $tokens:ident $span:ident) => { + $crate::__private::push_sub_eq_spanned(&mut $tokens, $span); + }; + + ($lifetime:lifetime $tokens:ident $span:ident) => { + $crate::__private::push_lifetime_spanned(&mut $tokens, $span, stringify!($lifetime)); + }; + + (_ $tokens:ident $span:ident) => { + $crate::__private::push_underscore_spanned(&mut $tokens, $span); + }; + + ($other:tt $tokens:ident $span:ident) => { + $crate::__private::parse_spanned(&mut $tokens, $span, stringify!($other)); + }; +} diff --git a/rust/quote/runtime.rs b/rust/quote/runtime.rs new file mode 100644 index 000000000..09a94f5dd --- /dev/null +++ b/rust/quote/runtime.rs @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use self::get_span::{GetSpan, GetSpanBase, GetSpanInner}; +use crate::{IdentFragment, ToTokens, TokenStreamExt}; +use core::fmt; +use core::iter; +use core::ops::BitOr; +use proc_macro2::{Group, Ident, Punct, Spacing, TokenTree}; + +#[doc(hidden)] +pub use alloc::format; +#[doc(hidden)] +pub use core::option::Option; + +#[doc(hidden)] +pub type Delimiter = proc_macro2::Delimiter; +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream = proc_macro2::TokenStream; + +#[doc(hidden)] +pub struct HasIterator; // True +#[doc(hidden)] +pub struct ThereIsNoIteratorInRepetition; // False + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = ThereIsNoIteratorInRepetition; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> ThereIsNoIteratorInRepetition { + ThereIsNoIteratorInRepetition + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> HasIterator { + HasIterator + } +} + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +/// Extension traits used by the implementation of `quote!`. These are defined +/// in separate traits, rather than as a single trait due to ambiguity issues. +/// +/// These traits expose a `quote_into_iter` method which should allow calling +/// whichever impl happens to be applicable. Calling that method repeatedly on +/// the returned value should be idempotent. +#[doc(hidden)] +pub mod ext { + use super::RepInterp; + use super::{HasIterator as HasIter, ThereIsNoIteratorInRepetition as DoesNotHaveIter}; + use crate::ToTokens; + use alloc::collections::btree_set::{self, BTreeSet}; + use core::slice; + + /// Extension trait providing the `quote_into_iter` method on iterators. + #[doc(hidden)] + pub trait RepIteratorExt: Iterator + Sized { + fn quote_into_iter(self) -> (Self, HasIter) { + (self, HasIter) + } + } + + impl RepIteratorExt for T {} + + /// Extension trait providing the `quote_into_iter` method for + /// non-iterable types. These types interpolate the same value in each + /// iteration of the repetition. + #[doc(hidden)] + pub trait RepToTokensExt { + /// Pretend to be an iterator for the purposes of `quote_into_iter`. + /// This allows repeated calls to `quote_into_iter` to continue + /// correctly returning DoesNotHaveIter. + fn next(&self) -> Option<&Self> { + Some(self) + } + + fn quote_into_iter(&self) -> (&Self, DoesNotHaveIter) { + (self, DoesNotHaveIter) + } + } + + impl RepToTokensExt for T {} + + /// Extension trait providing the `quote_into_iter` method for types that + /// can be referenced as an iterator. + #[doc(hidden)] + pub trait RepAsIteratorExt<'q> { + type Iter: Iterator; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter); + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &mut T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for [T] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q, const N: usize> RepAsIteratorExt<'q> for [T; N] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for Vec { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for BTreeSet { + type Iter = btree_set::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: RepAsIteratorExt<'q>> RepAsIteratorExt<'q> for RepInterp { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + self.0.quote_into_iter() + } + } +} + +// Helper type used within interpolations to allow for repeated binding names. +// Implements the relevant traits, and exports a dummy `next()` method. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct RepInterp(pub T); + +impl RepInterp { + // This method is intended to look like `Iterator::next`, and is called when + // a name is bound multiple times, as the previous binding will shadow the + // original `Iterator` object. This allows us to avoid advancing the + // iterator multiple times per iteration. + pub fn next(self) -> Option { + Some(self.0) + } +} + +impl Iterator for RepInterp { + type Item = T::Item; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +impl ToTokens for RepInterp { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens); + } +} + +#[doc(hidden)] +#[inline] +pub fn get_span(span: T) -> GetSpan { + GetSpan(GetSpanInner(GetSpanBase(span))) +} + +mod get_span { + use core::ops::Deref; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub struct GetSpan(pub(crate) GetSpanInner); + + pub struct GetSpanInner(pub(crate) GetSpanBase); + + pub struct GetSpanBase(pub(crate) T); + + impl GetSpan { + #[inline] + pub fn __into_span(self) -> Span { + ((self.0).0).0 + } + } + + impl GetSpanInner { + #[inline] + pub fn __into_span(&self) -> Span { + (self.0).0.join() + } + } + + impl GetSpanBase { + #[allow(clippy::unused_self)] + pub fn __into_span(&self) -> T { + unreachable!() + } + } + + impl Deref for GetSpan { + type Target = GetSpanInner; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl Deref for GetSpanInner { + type Target = GetSpanBase; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } +} + +#[doc(hidden)] +pub fn push_group(tokens: &mut TokenStream, delimiter: Delimiter, inner: TokenStream) { + tokens.append(Group::new(delimiter, inner)); +} + +#[doc(hidden)] +pub fn push_group_spanned( + tokens: &mut TokenStream, + span: Span, + delimiter: Delimiter, + inner: TokenStream, +) { + let mut g = Group::new(delimiter, inner); + g.set_span(span); + tokens.append(g); +} + +#[doc(hidden)] +pub fn parse(tokens: &mut TokenStream, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(iter::once(s)); +} + +#[doc(hidden)] +pub fn parse_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(s.into_iter().map(|t| respan_token_tree(t, span))); +} + +// Token tree with every span replaced by the given one. +fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree { + match &mut token { + TokenTree::Group(g) => { + let stream = g + .stream() + .into_iter() + .map(|token| respan_token_tree(token, span)) + .collect(); + *g = Group::new(g.delimiter(), stream); + g.set_span(span); + } + other => other.set_span(span), + } + token +} + +#[doc(hidden)] +pub fn push_ident(tokens: &mut TokenStream, s: &str) { + let span = Span::call_site(); + push_ident_spanned(tokens, span, s); +} + +#[doc(hidden)] +pub fn push_ident_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + tokens.append(ident_maybe_raw(s, span)); +} + +#[doc(hidden)] +pub fn push_lifetime(tokens: &mut TokenStream, lifetime: &str) { + tokens.extend([ + TokenTree::Punct(Punct::new('\'', Spacing::Joint)), + TokenTree::Ident(Ident::new(&lifetime[1..], Span::call_site())), + ]); +} + +#[doc(hidden)] +pub fn push_lifetime_spanned(tokens: &mut TokenStream, span: Span, lifetime: &str) { + tokens.extend([ + TokenTree::Punct({ + let mut apostrophe = Punct::new('\'', Spacing::Joint); + apostrophe.set_span(span); + apostrophe + }), + TokenTree::Ident(Ident::new(&lifetime[1..], span)), + ]); +} + +macro_rules! push_punct { + ($name:ident $spanned:ident $char1:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt $char3:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Joint)); + tokens.append(Punct::new($char3, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char3, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; +} + +push_punct!(push_add push_add_spanned '+'); +push_punct!(push_add_eq push_add_eq_spanned '+' '='); +push_punct!(push_and push_and_spanned '&'); +push_punct!(push_and_and push_and_and_spanned '&' '&'); +push_punct!(push_and_eq push_and_eq_spanned '&' '='); +push_punct!(push_at push_at_spanned '@'); +push_punct!(push_bang push_bang_spanned '!'); +push_punct!(push_caret push_caret_spanned '^'); +push_punct!(push_caret_eq push_caret_eq_spanned '^' '='); +push_punct!(push_colon push_colon_spanned ':'); +push_punct!(push_colon2 push_colon2_spanned ':' ':'); +push_punct!(push_comma push_comma_spanned ','); +push_punct!(push_div push_div_spanned '/'); +push_punct!(push_div_eq push_div_eq_spanned '/' '='); +push_punct!(push_dot push_dot_spanned '.'); +push_punct!(push_dot2 push_dot2_spanned '.' '.'); +push_punct!(push_dot3 push_dot3_spanned '.' '.' '.'); +push_punct!(push_dot_dot_eq push_dot_dot_eq_spanned '.' '.' '='); +push_punct!(push_eq push_eq_spanned '='); +push_punct!(push_eq_eq push_eq_eq_spanned '=' '='); +push_punct!(push_ge push_ge_spanned '>' '='); +push_punct!(push_gt push_gt_spanned '>'); +push_punct!(push_le push_le_spanned '<' '='); +push_punct!(push_lt push_lt_spanned '<'); +push_punct!(push_mul_eq push_mul_eq_spanned '*' '='); +push_punct!(push_ne push_ne_spanned '!' '='); +push_punct!(push_or push_or_spanned '|'); +push_punct!(push_or_eq push_or_eq_spanned '|' '='); +push_punct!(push_or_or push_or_or_spanned '|' '|'); +push_punct!(push_pound push_pound_spanned '#'); +push_punct!(push_question push_question_spanned '?'); +push_punct!(push_rarrow push_rarrow_spanned '-' '>'); +push_punct!(push_larrow push_larrow_spanned '<' '-'); +push_punct!(push_rem push_rem_spanned '%'); +push_punct!(push_rem_eq push_rem_eq_spanned '%' '='); +push_punct!(push_fat_arrow push_fat_arrow_spanned '=' '>'); +push_punct!(push_semi push_semi_spanned ';'); +push_punct!(push_shl push_shl_spanned '<' '<'); +push_punct!(push_shl_eq push_shl_eq_spanned '<' '<' '='); +push_punct!(push_shr push_shr_spanned '>' '>'); +push_punct!(push_shr_eq push_shr_eq_spanned '>' '>' '='); +push_punct!(push_star push_star_spanned '*'); +push_punct!(push_sub push_sub_spanned '-'); +push_punct!(push_sub_eq push_sub_eq_spanned '-' '='); + +#[doc(hidden)] +pub fn push_underscore(tokens: &mut TokenStream) { + push_underscore_spanned(tokens, Span::call_site()); +} + +#[doc(hidden)] +pub fn push_underscore_spanned(tokens: &mut TokenStream, span: Span) { + tokens.append(Ident::new("_", span)); +} + +// Helper method for constructing identifiers from the `format_ident!` macro, +// handling `r#` prefixes. +#[doc(hidden)] +pub fn mk_ident(id: &str, span: Option) -> Ident { + let span = span.unwrap_or_else(Span::call_site); + ident_maybe_raw(id, span) +} + +fn ident_maybe_raw(id: &str, span: Span) -> Ident { + if let Some(id) = id.strip_prefix("r#") { + Ident::new_raw(id, span) + } else { + Ident::new(id, span) + } +} + +// Adapts from `IdentFragment` to `fmt::Display` for use by the `format_ident!` +// macro, and exposes span information from these fragments. +// +// This struct also has forwarding implementations of the formatting traits +// `Octal`, `LowerHex`, `UpperHex`, and `Binary` to allow for their use within +// `format_ident!`. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct IdentFragmentAdapter(pub T); + +impl IdentFragmentAdapter { + pub fn span(&self) -> Option { + self.0.span() + } +} + +impl fmt::Display for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(&self.0, f) + } +} + +impl fmt::Octal for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Octal::fmt(&self.0, f) + } +} + +impl fmt::LowerHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} + +impl fmt::UpperHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl fmt::Binary for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Binary::fmt(&self.0, f) + } +} diff --git a/rust/quote/spanned.rs b/rust/quote/spanned.rs new file mode 100644 index 000000000..54ce9177f --- /dev/null +++ b/rust/quote/spanned.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::ToTokens; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Span, TokenStream}; + +// Not public API other than via the syn crate. Use syn::spanned::Spanned. +pub trait Spanned: private::Sealed { + fn __span(&self) -> Span; +} + +impl Spanned for Span { + fn __span(&self) -> Span { + *self + } +} + +impl Spanned for DelimSpan { + fn __span(&self) -> Span { + self.join() + } +} + +impl Spanned for T { + fn __span(&self) -> Span { + join_spans(self.into_token_stream()) + } +} + +fn join_spans(tokens: TokenStream) -> Span { + let mut iter = tokens.into_iter().map(|tt| tt.span()); + + let first = match iter.next() { + Some(span) => span, + None => return Span::call_site(), + }; + + iter.fold(None, |_prev, next| Some(next)) + .and_then(|last| first.join(last)) + .unwrap_or(first) +} + +mod private { + use crate::ToTokens; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub trait Sealed {} + impl Sealed for Span {} + impl Sealed for DelimSpan {} + impl Sealed for T {} +} diff --git a/rust/quote/to_tokens.rs b/rust/quote/to_tokens.rs new file mode 100644 index 000000000..1af1089e1 --- /dev/null +++ b/rust/quote/to_tokens.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use super::TokenStreamExt; +use alloc::borrow::Cow; +use alloc::rc::Rc; +use core::iter; +use proc_macro2::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree}; +use std::ffi::{CStr, CString}; + +/// Types that can be interpolated inside a `quote!` invocation. +pub trait ToTokens { + /// Write `self` to the given `TokenStream`. + /// + /// The token append methods provided by the [`TokenStreamExt`] extension + /// trait may be useful for implementing `ToTokens`. + /// + /// # Example + /// + /// Example implementation for a struct representing Rust paths like + /// `std::cmp::PartialEq`: + /// + /// ``` + /// use proc_macro2::{TokenTree, Spacing, Span, Punct, TokenStream}; + /// use quote::{TokenStreamExt, ToTokens}; + /// + /// pub struct Path { + /// pub global: bool, + /// pub segments: Vec, + /// } + /// + /// impl ToTokens for Path { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// for (i, segment) in self.segments.iter().enumerate() { + /// if i > 0 || self.global { + /// // Double colon `::` + /// tokens.append(Punct::new(':', Spacing::Joint)); + /// tokens.append(Punct::new(':', Spacing::Alone)); + /// } + /// segment.to_tokens(tokens); + /// } + /// } + /// } + /// # + /// # pub struct PathSegment; + /// # + /// # impl ToTokens for PathSegment { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # unimplemented!() + /// # } + /// # } + /// ``` + fn to_tokens(&self, tokens: &mut TokenStream); + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn to_token_stream(&self) -> TokenStream { + let mut tokens = TokenStream::new(); + self.to_tokens(&mut tokens); + tokens + } + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn into_token_stream(self) -> TokenStream + where + Self: Sized, + { + self.to_token_stream() + } +} + +impl ToTokens for &T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for &mut T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Box { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Rc { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Option { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Some(t) = self { + t.to_tokens(tokens); + } + } +} + +impl ToTokens for str { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::string(self)); + } +} + +impl ToTokens for String { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.as_str().to_tokens(tokens); + } +} + +impl ToTokens for i8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i8_suffixed(*self)); + } +} + +impl ToTokens for i16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i16_suffixed(*self)); + } +} + +impl ToTokens for i32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i32_suffixed(*self)); + } +} + +impl ToTokens for i64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i64_suffixed(*self)); + } +} + +impl ToTokens for i128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i128_suffixed(*self)); + } +} + +impl ToTokens for isize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::isize_suffixed(*self)); + } +} + +impl ToTokens for u8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u8_suffixed(*self)); + } +} + +impl ToTokens for u16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u16_suffixed(*self)); + } +} + +impl ToTokens for u32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u32_suffixed(*self)); + } +} + +impl ToTokens for u64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u64_suffixed(*self)); + } +} + +impl ToTokens for u128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u128_suffixed(*self)); + } +} + +impl ToTokens for usize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::usize_suffixed(*self)); + } +} + +impl ToTokens for f32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f32_suffixed(*self)); + } +} + +impl ToTokens for f64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f64_suffixed(*self)); + } +} + +impl ToTokens for char { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::character(*self)); + } +} + +impl ToTokens for bool { + fn to_tokens(&self, tokens: &mut TokenStream) { + let word = if *self { "true" } else { "false" }; + tokens.append(Ident::new(word, Span::call_site())); + } +} + +impl ToTokens for CStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for CString { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for Group { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Ident { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Punct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Literal { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenTree { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenStream { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(iter::once(self.clone())); + } + + fn into_token_stream(self) -> TokenStream { + self + } +} diff --git a/rust/syn/README.md b/rust/syn/README.md new file mode 100644 index 000000000..d3e3981da --- /dev/null +++ b/rust/syn/README.md @@ -0,0 +1,13 @@ +# `syn` + +These source files come from the Rust `syn` crate, version 2.0.106 +(released 2025-08-16), hosted in the +repository, licensed under "Apache-2.0 OR MIT" and only modified to add +the SPDX license identifiers and to remove the `unicode-ident` +dependency. + +For copyright details, please see: + + https://github.com/dtolnay/syn/blob/2.0.106/README.md#license + https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-APACHE + https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-MIT diff --git a/rust/syn/attr.rs b/rust/syn/attr.rs new file mode 100644 index 000000000..020b31f9d --- /dev/null +++ b/rust/syn/attr.rs @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "parsing")] +use crate::error::Error; +#[cfg(feature = "parsing")] +use crate::error::Result; +use crate::expr::Expr; +use crate::mac::MacroDelimiter; +#[cfg(feature = "parsing")] +use crate::meta::{self, ParseNestedMeta}; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream, Parser}; +use crate::path::Path; +use crate::token; +use proc_macro2::TokenStream; +#[cfg(feature = "printing")] +use std::iter; +#[cfg(feature = "printing")] +use std::slice; + +ast_struct! { + /// An attribute, like `#[repr(transparent)]`. + /// + ///
+ /// + /// # Syntax + /// + /// Rust has six types of attributes. + /// + /// - Outer attributes like `#[repr(transparent)]`. These appear outside or + /// in front of the item they describe. + /// + /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside + /// of the item they describe, usually a module. + /// + /// - Outer one-line doc comments like `/// Example`. + /// + /// - Inner one-line doc comments like `//! Please file an issue`. + /// + /// - Outer documentation blocks `/** Example */`. + /// + /// - Inner documentation blocks `/*! Please file an issue */`. + /// + /// The `style` field of type `AttrStyle` distinguishes whether an attribute + /// is outer or inner. + /// + /// Every attribute has a `path` that indicates the intended interpretation + /// of the rest of the attribute's contents. The path and the optional + /// additional contents are represented together in the `meta` field of the + /// attribute in three possible varieties: + /// + /// - Meta::Path — attributes whose information content conveys just a + /// path, for example the `#[test]` attribute. + /// + /// - Meta::List — attributes that carry arbitrary tokens after the + /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For + /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. + /// + /// - Meta::NameValue — attributes with an `=` sign after the path, + /// followed by a Rust expression. For example `#[path = + /// "sys/windows.rs"]`. + /// + /// All doc comments are represented in the NameValue style with a path of + /// "doc", as this is how they are processed by the compiler and by + /// `macro_rules!` macros. + /// + /// ```text + /// #[derive(Copy, Clone)] + /// ~~~~~~Path + /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// + /// #[path = "sys/windows.rs"] + /// ~~~~Path + /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue + /// + /// #[test] + /// ^^^^Meta::Path + /// ``` + /// + ///
+ /// + /// # Parsing from tokens to Attribute + /// + /// This type does not implement the [`Parse`] trait and thus cannot be + /// parsed directly by [`ParseStream::parse`]. Instead use + /// [`ParseStream::call`] with one of the two parser functions + /// [`Attribute::parse_outer`] or [`Attribute::parse_inner`] depending on + /// which you intend to parse. + /// + /// [`Parse`]: crate::parse::Parse + /// [`ParseStream::parse`]: crate::parse::ParseBuffer::parse + /// [`ParseStream::call`]: crate::parse::ParseBuffer::call + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + /// + ///


+ /// + /// # Parsing from Attribute to structured arguments + /// + /// The grammar of attributes in Rust is very flexible, which makes the + /// syntax tree not that useful on its own. In particular, arguments of the + /// `Meta::List` variety of attribute are held in an arbitrary `tokens: + /// TokenStream`. Macros are expected to check the `path` of the attribute, + /// decide whether they recognize it, and then parse the remaining tokens + /// according to whatever grammar they wish to require for that kind of + /// attribute. Use [`parse_args()`] to parse those tokens into the expected + /// data structure. + /// + /// [`parse_args()`]: Attribute::parse_args + /// + ///


+ /// + /// # Doc comments + /// + /// The compiler transforms doc comments, such as `/// comment` and `/*! + /// comment */`, into attributes before macros are expanded. Each comment is + /// expanded into an attribute of the form `#[doc = r"comment"]`. + /// + /// As an example, the following `mod` items are expanded identically: + /// + /// ``` + /// # use syn::{ItemMod, parse_quote}; + /// let doc: ItemMod = parse_quote! { + /// /// Single line doc comments + /// /// We write so many! + /// /** + /// * Multi-line comments... + /// * May span many lines + /// */ + /// mod example { + /// //! Of course, they can be inner too + /// /*! And fit in a single line */ + /// } + /// }; + /// let attr: ItemMod = parse_quote! { + /// #[doc = r" Single line doc comments"] + /// #[doc = r" We write so many!"] + /// #[doc = r" + /// * Multi-line comments... + /// * May span many lines + /// "] + /// mod example { + /// #![doc = r" Of course, they can be inner too"] + /// #![doc = r" And fit in a single line "] + /// } + /// }; + /// assert_eq!(doc, attr); + /// ``` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Attribute { + pub pound_token: Token![#], + pub style: AttrStyle, + pub bracket_token: token::Bracket, + pub meta: Meta, + } +} + +impl Attribute { + /// Returns the path that identifies the interpretation of this attribute. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + self.meta.path() + } + + /// Parse the arguments to the attribute as a syntax tree. + /// + /// This is similar to pulling out the `TokenStream` from `Meta::List` and + /// doing `syn::parse2::(meta_list.tokens)`, except that using + /// `parse_args` the error message has a more useful span when `tokens` is + /// empty. + /// + /// The surrounding delimiters are *not* included in the input to the + /// parser. + /// + /// ```text + /// #[my_attr(value < 5)] + /// ^^^^^^^^^ what gets parsed + /// ``` + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, Expr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[precondition(value < 5)] + /// }; + /// + /// if attr.path().is_ident("precondition") { + /// let precondition: Expr = attr.parse_args()?; + /// // ... + /// } + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// Parse the arguments to the attribute using the given parser. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] + /// }; + /// + /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; + /// + /// // Attribute does not have a Parse impl, so we couldn't directly do: + /// // let bwom: Attribute = attr.parse_args()?; + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + match &self.meta { + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new( + meta.eq_token.span, + format_args!( + "expected parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(&meta.path), + ), + )), + Meta::List(meta) => meta.parse_args_with(parser), + } + } + + /// Parse the arguments to the attribute, expecting it to follow the + /// conventional structure used by most of Rust's built-in attributes. + /// + /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference + /// explains the convention in more detail. Not all attributes follow this + /// convention, so [`parse_args()`][Self::parse_args] is available if you + /// need to parse arbitrarily goofy attribute syntax. + /// + /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax + /// + /// # Example + /// + /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute + /// syntax. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; + /// + /// let input: ItemStruct = parse_quote! { + /// #[repr(C, align(4))] + /// pub struct MyStruct(u16, u32); + /// }; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// // #[repr(C)] + /// if meta.path.is_ident("C") { + /// repr_c = true; + /// return Ok(()); + /// } + /// + /// // #[repr(transparent)] + /// if meta.path.is_ident("transparent") { + /// repr_transparent = true; + /// return Ok(()); + /// } + /// + /// // #[repr(align(N))] + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// return Ok(()); + /// } + /// + /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 + /// if meta.path.is_ident("packed") { + /// if meta.input.peek(token::Paren) { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_packed = Some(n); + /// } else { + /// repr_packed = Some(1); + /// } + /// return Ok(()); + /// } + /// + /// Err(meta.error("unrecognized repr")) + /// })?; + /// } + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Alternatives + /// + /// In some cases, for attributes which have nested layers of structured + /// content, the following less flexible approach might be more convenient: + /// + /// ``` + /// # use syn::{parse_quote, ItemStruct}; + /// # + /// # let input: ItemStruct = parse_quote! { + /// # #[repr(C, align(4))] + /// # pub struct MyStruct(u16, u32); + /// # }; + /// # + /// use syn::punctuated::Punctuated; + /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// let nested = attr.parse_args_with(Punctuated::::parse_terminated)?; + /// for meta in nested { + /// match meta { + /// // #[repr(C)] + /// Meta::Path(path) if path.is_ident("C") => { + /// repr_c = true; + /// } + /// + /// // #[repr(align(N))] + /// Meta::List(meta) if meta.path.is_ident("align") => { + /// let lit: LitInt = meta.parse_args()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// } + /// + /// /* ... */ + /// + /// _ => { + /// return Err(Error::new_spanned(meta, "unrecognized repr")); + /// } + /// } + /// } + /// } + /// } + /// # Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } + + /// Parses zero or more outer attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_outer(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) { + attrs.push(input.call(parsing::single_parse_outer)?); + } + Ok(attrs) + } + + /// Parses zero or more inner attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_inner(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + parsing::parse_inner(input, &mut attrs)?; + Ok(attrs) + } +} + +ast_enum! { + /// Distinguishes between attributes that decorate an item and attributes + /// that are contained within an item. + /// + /// # Outer attributes + /// + /// - `#[repr(transparent)]` + /// - `/// # Example` + /// - `/** Please file an issue */` + /// + /// # Inner attributes + /// + /// - `#![feature(proc_macro)]` + /// - `//! # Example` + /// - `/*! Please file an issue */` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum AttrStyle { + Outer, + Inner(Token![!]), + } +} + +ast_enum! { + /// Content of a compile-time structured attribute. + /// + /// ## Path + /// + /// A meta path is like the `test` in `#[test]`. + /// + /// ## List + /// + /// A meta list is like the `derive(Copy)` in `#[derive(Copy)]`. + /// + /// ## NameValue + /// + /// A name-value meta is like the `path = "..."` in `#[path = + /// "sys/windows.rs"]`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Meta { + Path(Path), + + /// A structured list within an attribute, like `derive(Copy, Clone)`. + List(MetaList), + + /// A name-value pair within an attribute, like `feature = "nightly"`. + NameValue(MetaNameValue), + } +} + +ast_struct! { + /// A structured list within an attribute, like `derive(Copy, Clone)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaList { + pub path: Path, + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, + } +} + +ast_struct! { + /// A name-value pair within an attribute, like `feature = "nightly"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaNameValue { + pub path: Path, + pub eq_token: Token![=], + pub value: Expr, + } +} + +impl Meta { + /// Returns the path that begins this structured meta item. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + match self { + Meta::Path(path) => path, + Meta::List(meta) => &meta.path, + Meta::NameValue(meta) => &meta.path, + } + } + + /// Error if this is a `Meta::List` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_path_only(&self) -> Result<&Path> { + let error_span = match self { + Meta::Path(path) => return Ok(path), + Meta::List(meta) => meta.delimiter.span().open(), + Meta::NameValue(meta) => meta.eq_token.span, + }; + Err(Error::new(error_span, "unexpected token in attribute")) + } + + /// Error if this is a `Meta::Path` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_list(&self) -> Result<&MetaList> { + match self { + Meta::List(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: `{}(...)`", + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), + } + } + + /// Error if this is a `Meta::Path` or `Meta::List`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_name_value(&self) -> Result<&MetaNameValue> { + match self { + Meta::NameValue(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected a value for this attribute: `{} = ...`", + parsing::DisplayPath(path), + ), + )), + Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), + } + } +} + +impl MetaList { + /// See [`Attribute::parse_args`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// See [`Attribute::parse_args_with`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } + + /// See [`Attribute::parse_nested_meta`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } +} + +#[cfg(feature = "printing")] +pub(crate) trait FilterAttrs<'a> { + type Ret: Iterator; + + fn outer(self) -> Self::Ret; + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret; +} + +#[cfg(feature = "printing")] +impl<'a> FilterAttrs<'a> for &'a [Attribute] { + type Ret = iter::Filter, fn(&&Attribute) -> bool>; + + fn outer(self) -> Self::Ret { + fn is_outer(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Outer => true, + AttrStyle::Inner(_) => false, + } + } + self.iter().filter(is_outer) + } + + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret { + fn is_inner(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Inner(_) => true, + AttrStyle::Outer => false, + } + } + self.iter().filter(is_inner) + } +} + +impl From for Meta { + fn from(meta: Path) -> Meta { + Meta::Path(meta) + } +} + +impl From for Meta { + fn from(meta: MetaList) -> Meta { + Meta::List(meta) + } +} + +impl From for Meta { + fn from(meta: MetaNameValue) -> Meta { + Meta::NameValue(meta) + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::error::Result; + use crate::expr::{Expr, ExprLit}; + use crate::lit::Lit; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + use crate::{mac, token}; + use proc_macro2::Ident; + use std::fmt::{self, Display}; + + pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { + while input.peek(Token![#]) && input.peek2(Token![!]) { + attrs.push(input.call(single_parse_inner)?); + } + Ok(()) + } + + pub(crate) fn single_parse_inner(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Inner(input.parse()?), + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + pub(crate) fn single_parse_outer(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Outer, + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Meta { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaList { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_list_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaNameValue { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_name_value_after_path(path, input) + } + } + + // Unlike meta::parse_meta_path which accepts arbitrary keywords in the path, + // only the `unsafe` keyword is accepted as an attribute's outermost path. + fn parse_outermost_meta_path(input: ParseStream) -> Result { + if input.peek(Token![unsafe]) { + let unsafe_token: Token![unsafe] = input.parse()?; + Ok(Path::from(Ident::new("unsafe", unsafe_token.span))) + } else { + Path::parse_mod_style(input) + } + } + + pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { + parse_meta_list_after_path(path, input).map(Meta::List) + } else if input.peek(Token![=]) { + parse_meta_name_value_after_path(path, input).map(Meta::NameValue) + } else { + Ok(Meta::Path(path)) + } + } + + fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { + let (delimiter, tokens) = mac::parse_delimiter(input)?; + Ok(MetaList { + path, + delimiter, + tokens, + }) + } + + fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { + let eq_token: Token![=] = input.parse()?; + let ahead = input.fork(); + let lit: Option = ahead.parse()?; + let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { + input.advance_to(&ahead); + Expr::Lit(ExprLit { + attrs: Vec::new(), + lit, + }) + } else if input.peek(Token![#]) && input.peek2(token::Bracket) { + return Err(input.error("unexpected attribute inside of attribute")); + } else { + input.parse()? + }; + Ok(MetaNameValue { + path, + eq_token, + value, + }) + } + + pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); + + impl<'a> Display for DisplayAttrStyle<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(match self.0 { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }) + } + } + + pub(super) struct DisplayPath<'a>(pub &'a Path); + + impl<'a> Display for DisplayPath<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + for (i, segment) in self.0.segments.iter().enumerate() { + if i > 0 || self.0.leading_colon.is_some() { + formatter.write_str("::")?; + } + write!(formatter, "{}", segment.ident)?; + } + Ok(()) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::path; + use crate::path::printing::PathStyle; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Attribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pound_token.to_tokens(tokens); + if let AttrStyle::Inner(b) = &self.style { + b.to_tokens(tokens); + } + self.bracket_token.surround(tokens, |tokens| { + self.meta.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Meta { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Meta::Path(path) => path::printing::print_path(tokens, path, PathStyle::Mod), + Meta::List(meta_list) => meta_list.to_tokens(tokens), + Meta::NameValue(meta_name_value) => meta_name_value.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaList { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.delimiter.surround(tokens, self.tokens.clone()); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaNameValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.eq_token.to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} diff --git a/rust/syn/bigint.rs b/rust/syn/bigint.rs new file mode 100644 index 000000000..d706f9f51 --- /dev/null +++ b/rust/syn/bigint.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::ops::{AddAssign, MulAssign}; + +// For implementing base10_digits() accessor on LitInt. +pub(crate) struct BigInt { + digits: Vec, +} + +impl BigInt { + pub(crate) fn new() -> Self { + BigInt { digits: Vec::new() } + } + + pub(crate) fn to_string(&self) -> String { + let mut repr = String::with_capacity(self.digits.len()); + + let mut has_nonzero = false; + for digit in self.digits.iter().rev() { + has_nonzero |= *digit != 0; + if has_nonzero { + repr.push((*digit + b'0') as char); + } + } + + if repr.is_empty() { + repr.push('0'); + } + + repr + } + + fn reserve_two_digits(&mut self) { + let len = self.digits.len(); + let desired = + len + !self.digits.ends_with(&[0, 0]) as usize + !self.digits.ends_with(&[0]) as usize; + self.digits.resize(desired, 0); + } +} + +impl AddAssign for BigInt { + // Assumes increment <16. + fn add_assign(&mut self, mut increment: u8) { + self.reserve_two_digits(); + + let mut i = 0; + while increment > 0 { + let sum = self.digits[i] + increment; + self.digits[i] = sum % 10; + increment = sum / 10; + i += 1; + } + } +} + +impl MulAssign for BigInt { + // Assumes base <=16. + fn mul_assign(&mut self, base: u8) { + self.reserve_two_digits(); + + let mut carry = 0; + for digit in &mut self.digits { + let prod = *digit * base + carry; + *digit = prod % 10; + carry = prod / 10; + } + } +} diff --git a/rust/syn/buffer.rs b/rust/syn/buffer.rs new file mode 100644 index 000000000..406f70db3 --- /dev/null +++ b/rust/syn/buffer.rs @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A stably addressed token buffer supporting efficient traversal based on a +//! cheaply copyable cursor. + +// This module is heavily commented as it contains most of the unsafe code in +// Syn, and caution should be used when editing it. The public-facing interface +// is 100% safe but the implementation is fragile internally. + +use crate::Lifetime; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::cmp::Ordering; +use std::marker::PhantomData; +use std::ptr; + +/// Internal type which is used instead of `TokenTree` to represent a token tree +/// within a `TokenBuffer`. +enum Entry { + // Mimicking types from proc-macro. + // Group entries contain the offset to the matching End entry. + Group(Group, usize), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain the offset (negative) to the start of the buffer, and + // offset (negative) to the matching Group entry. + End(isize, isize), +} + +/// A buffer that can be efficiently traversed multiple times, unlike +/// `TokenStream` which requires a deep copy in order to traverse more than +/// once. +pub struct TokenBuffer { + // NOTE: Do not implement clone on this - while the current design could be + // cloned, other designs which could be desirable may not be cloneable. + entries: Box<[Entry]>, +} + +impl TokenBuffer { + fn recursive_new(entries: &mut Vec, stream: TokenStream) { + for tt in stream { + match tt { + TokenTree::Ident(ident) => entries.push(Entry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(Entry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(Entry::Literal(literal)), + TokenTree::Group(group) => { + let group_start_index = entries.len(); + entries.push(Entry::End(0, 0)); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + let group_offset = group_end_index - group_start_index; + entries.push(Entry::End( + -(group_end_index as isize), + -(group_offset as isize), + )); + entries[group_start_index] = Entry::Group(group, group_offset); + } + } + } + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro::TokenStream`. + #[cfg(feature = "proc-macro")] + #[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] + pub fn new(stream: proc_macro::TokenStream) -> Self { + Self::new2(stream.into()) + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro2::TokenStream`. + pub fn new2(stream: TokenStream) -> Self { + let mut entries = Vec::new(); + Self::recursive_new(&mut entries, stream); + entries.push(Entry::End(-(entries.len() as isize), 0)); + Self { + entries: entries.into_boxed_slice(), + } + } + + /// Creates a cursor referencing the first token in the buffer and able to + /// traverse until the end of the buffer. + pub fn begin(&self) -> Cursor { + let ptr = self.entries.as_ptr(); + unsafe { Cursor::create(ptr, ptr.add(self.entries.len() - 1)) } + } +} + +/// A cheaply copyable cursor into a `TokenBuffer`. +/// +/// This cursor holds a shared reference into the immutable data which is used +/// internally to represent a `TokenStream`, and can be efficiently manipulated +/// and copied around. +/// +/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` +/// object and get a cursor to its first token with `begin()`. +pub struct Cursor<'a> { + // The current entry which the `Cursor` is pointing at. + ptr: *const Entry, + // This is the only `Entry::End` object which this cursor is allowed to + // point at. All other `End` objects are skipped over in `Cursor::create`. + scope: *const Entry, + // Cursor is covariant in 'a. This field ensures that our pointers are still + // valid. + marker: PhantomData<&'a Entry>, +} + +impl<'a> Cursor<'a> { + /// Creates a cursor referencing a static empty TokenStream. + pub fn empty() -> Self { + // It's safe in this situation for us to put an `Entry` object in global + // storage, despite it not actually being safe to send across threads + // (`Ident` is a reference into a thread-local table). This is because + // this entry never includes a `Ident` object. + // + // This wrapper struct allows us to break the rules and put a `Sync` + // object in global storage. + struct UnsafeSyncEntry(Entry); + unsafe impl Sync for UnsafeSyncEntry {} + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0, 0)); + + Cursor { + ptr: &EMPTY_ENTRY.0, + scope: &EMPTY_ENTRY.0, + marker: PhantomData, + } + } + + /// This create method intelligently exits non-explicitly-entered + /// `None`-delimited scopes when the cursor reaches the end of them, + /// allowing for them to be treated transparently. + unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { + // NOTE: If we're looking at a `End`, we want to advance the cursor + // past it, unless `ptr == scope`, which means that we're at the edge of + // our cursor's scope. We should only have `ptr != scope` at the exit + // from None-delimited groups entered with `ignore_none`. + while let Entry::End(..) = unsafe { &*ptr } { + if ptr::eq(ptr, scope) { + break; + } + ptr = unsafe { ptr.add(1) }; + } + + Cursor { + ptr, + scope, + marker: PhantomData, + } + } + + /// Get the current entry. + fn entry(self) -> &'a Entry { + unsafe { &*self.ptr } + } + + /// Bump the cursor to point at the next token after the current one. This + /// is undefined behavior if the cursor is currently looking at an + /// `Entry::End`. + /// + /// If the cursor is looking at an `Entry::Group`, the bumped cursor will + /// point at the first token in the group (with the same scope end). + unsafe fn bump_ignore_group(self) -> Cursor<'a> { + unsafe { Cursor::create(self.ptr.offset(1), self.scope) } + } + + /// While the cursor is looking at a `None`-delimited group, move it to look + /// at the first token inside instead. If the group is empty, this will move + /// the cursor past the `None`-delimited group. + /// + /// WARNING: This mutates its argument. + fn ignore_none(&mut self) { + while let Entry::Group(group, _) = self.entry() { + if group.delimiter() == Delimiter::None { + unsafe { *self = self.bump_ignore_group() }; + } else { + break; + } + } + } + + /// Checks whether the cursor is currently pointing at the end of its valid + /// scope. + pub fn eof(self) -> bool { + // We're at eof if we're at the end of our scope. + ptr::eq(self.ptr, self.scope) + } + + /// If the cursor is pointing at a `Ident`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Punct`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() != '\'' => { + Some((punct.clone(), unsafe { self.bump_ignore_group() })) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Literal`, return it along with a cursor + /// pointing at the next `TokenTree`. + pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Lifetime`, returns it along with a + /// cursor pointing at the next `TokenTree`. + pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + let next = unsafe { self.bump_ignore_group() }; + let (ident, rest) = next.ident()?; + let lifetime = Lifetime { + apostrophe: punct.span(), + ident, + }; + Some((lifetime, rest)) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Group` with the given delimiter, returns + /// a cursor into that group and one pointing to the next `TokenTree`. + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { + // If we're not trying to enter a none-delimited group, we want to + // ignore them. We have to make sure to _not_ ignore them when we want + // to enter them, of course. For obvious reasons. + if delim != Delimiter::None { + self.ignore_none(); + } + + if let Entry::Group(group, end_offset) = self.entry() { + if group.delimiter() == delim { + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, span, after_group)); + } + } + + None + } + + /// If the cursor is pointing at a `Group`, returns a cursor into the group + /// and one pointing to the next `TokenTree`. + pub fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let delimiter = group.delimiter(); + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, delimiter, span, after_group)); + } + + None + } + + pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((group.clone(), after_group)); + } + + None + } + + /// Copies all remaining tokens visible from this cursor into a + /// `TokenStream`. + pub fn token_stream(self) -> TokenStream { + let mut tts = Vec::new(); + let mut cursor = self; + while let Some((tt, rest)) = cursor.token_tree() { + tts.push(tt); + cursor = rest; + } + tts.into_iter().collect() + } + + /// If the cursor is pointing at a `TokenTree`, returns it along with a + /// cursor pointing at the next `TokenTree`. + /// + /// Returns `None` if the cursor has reached the end of its stream. + /// + /// This method does not treat `None`-delimited groups as transparent, and + /// will return a `Group(None, ..)` if the cursor is looking at one. + pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { + let (tree, len) = match self.entry() { + Entry::Group(group, end_offset) => (group.clone().into(), *end_offset), + Entry::Literal(literal) => (literal.clone().into(), 1), + Entry::Ident(ident) => (ident.clone().into(), 1), + Entry::Punct(punct) => (punct.clone().into(), 1), + Entry::End(..) => return None, + }; + + let rest = unsafe { Cursor::create(self.ptr.add(len), self.scope) }; + Some((tree, rest)) + } + + /// Returns the `Span` of the current token, or `Span::call_site()` if this + /// cursor points to eof. + pub fn span(mut self) -> Span { + match self.entry() { + Entry::Group(group, _) => group.span(), + Entry::Literal(literal) => literal.span(), + Entry::Ident(ident) => ident.span(), + Entry::Punct(punct) => punct.span(), + Entry::End(_, offset) => { + self.ptr = unsafe { self.ptr.offset(*offset) }; + if let Entry::Group(group, _) = self.entry() { + group.span_close() + } else { + Span::call_site() + } + } + } + } + + /// Returns the `Span` of the token immediately prior to the position of + /// this cursor, or of the current token if there is no previous one. + #[cfg(any(feature = "full", feature = "derive"))] + pub(crate) fn prev_span(mut self) -> Span { + if start_of_buffer(self) < self.ptr { + self.ptr = unsafe { self.ptr.offset(-1) }; + } + self.span() + } + + /// Skip over the next token that is not a None-delimited group, without + /// cloning it. Returns `None` if this cursor points to eof. + /// + /// This method treats `'lifetimes` as a single token. + pub(crate) fn skip(mut self) -> Option> { + self.ignore_none(); + + let len = match self.entry() { + Entry::End(..) => return None, + + // Treat lifetimes as a single tt for the purposes of 'skip'. + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + match unsafe { &*self.ptr.add(1) } { + Entry::Ident(_) => 2, + _ => 1, + } + } + + Entry::Group(_, end_offset) => *end_offset, + _ => 1, + }; + + Some(unsafe { Cursor::create(self.ptr.add(len), self.scope) }) + } + + pub(crate) fn scope_delimiter(self) -> Delimiter { + match unsafe { &*self.scope } { + Entry::End(_, offset) => match unsafe { &*self.scope.offset(*offset) } { + Entry::Group(group, _) => group.delimiter(), + _ => Delimiter::None, + }, + _ => unreachable!(), + } + } +} + +impl<'a> Copy for Cursor<'a> {} + +impl<'a> Clone for Cursor<'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a> Eq for Cursor<'a> {} + +impl<'a> PartialEq for Cursor<'a> { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.ptr, other.ptr) + } +} + +impl<'a> PartialOrd for Cursor<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + if same_buffer(*self, *other) { + Some(cmp_assuming_same_buffer(*self, *other)) + } else { + None + } + } +} + +pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { + ptr::eq(a.scope, b.scope) +} + +pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { + ptr::eq(start_of_buffer(a), start_of_buffer(b)) +} + +fn start_of_buffer(cursor: Cursor) -> *const Entry { + unsafe { + match &*cursor.scope { + Entry::End(offset, _) => cursor.scope.offset(*offset), + _ => unreachable!(), + } + } +} + +pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { + a.ptr.cmp(&b.ptr) +} + +pub(crate) fn open_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_open(), + _ => cursor.span(), + } +} diff --git a/rust/syn/classify.rs b/rust/syn/classify.rs new file mode 100644 index 000000000..92a975bfe --- /dev/null +++ b/rust/syn/classify.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "full")] +use crate::expr::Expr; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::generics::TypeParamBound; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::path::{Path, PathArguments}; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::punctuated::Punctuated; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::ty::{ReturnType, Type}; +#[cfg(feature = "full")] +use proc_macro2::{Delimiter, TokenStream, TokenTree}; +#[cfg(any(feature = "printing", feature = "full"))] +use std::ops::ControlFlow; + +#[cfg(feature = "full")] +pub(crate) fn requires_semi_to_be_stmt(expr: &Expr) -> bool { + match expr { + Expr::Macro(expr) => !expr.mac.delimiter.is_brace(), + _ => requires_comma_to_be_match_arm(expr), + } +} + +#[cfg(feature = "full")] +pub(crate) fn requires_comma_to_be_match_arm(expr: &Expr) -> bool { + match expr { + Expr::If(_) + | Expr::Match(_) + | Expr::Block(_) | Expr::Unsafe(_) // both under ExprKind::Block in rustc + | Expr::While(_) + | Expr::Loop(_) + | Expr::ForLoop(_) + | Expr::TryBlock(_) + | Expr::Const(_) => false, + + Expr::Array(_) + | Expr::Assign(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Binary(_) + | Expr::Break(_) + | Expr::Call(_) + | Expr::Cast(_) + | Expr::Closure(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Yield(_) + | Expr::Verbatim(_) => true, + } +} + +#[cfg(feature = "printing")] +pub(crate) fn trailing_unparameterized_path(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Path(t) => match last_type_in_path(&t.path) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Macro(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) + | Type::Verbatim(_) => return false, + } + } + + fn last_type_in_path(path: &Path) -> ControlFlow { + match &path.segments.last().unwrap().arguments { + PathArguments::None => ControlFlow::Break(true), + PathArguments::AngleBracketed(_) => ControlFlow::Break(false), + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => ControlFlow::Break(false), + ReturnType::Type(_, ret) => ControlFlow::Continue(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => last_type_in_path(&t.path), + TypeParamBound::Lifetime(_) + | TypeParamBound::PreciseCapture(_) + | TypeParamBound::Verbatim(_) => ControlFlow::Break(false), + } + } +} + +/// Whether the expression's first token is the label of a loop/block. +#[cfg(all(feature = "printing", feature = "full"))] +pub(crate) fn expr_leading_label(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Block(e) => return e.label.is_some(), + Expr::ForLoop(e) => return e.label.is_some(), + Expr::Loop(e) => return e.label.is_some(), + Expr::While(e) => return e.label.is_some(), + + Expr::Assign(e) => expr = &e.left, + Expr::Await(e) => expr = &e.base, + Expr::Binary(e) => expr = &e.left, + Expr::Call(e) => expr = &e.func, + Expr::Cast(e) => expr = &e.expr, + Expr::Field(e) => expr = &e.base, + Expr::Index(e) => expr = &e.expr, + Expr::MethodCall(e) => expr = &e.receiver, + Expr::Range(e) => match &e.start { + Some(start) => expr = start, + None => return false, + }, + Expr::Try(e) => expr = &e.expr, + + Expr::Array(_) + | Expr::Async(_) + | Expr::Break(_) + | Expr::Closure(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Unsafe(_) + | Expr::Verbatim(_) + | Expr::Yield(_) => return false, + } + } +} + +/// Whether the expression's last token is `}`. +#[cfg(feature = "full")] +pub(crate) fn expr_trailing_brace(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Async(_) + | Expr::Block(_) + | Expr::Const(_) + | Expr::ForLoop(_) + | Expr::If(_) + | Expr::Loop(_) + | Expr::Match(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Unsafe(_) + | Expr::While(_) => return true, + + Expr::Assign(e) => expr = &e.right, + Expr::Binary(e) => expr = &e.right, + Expr::Break(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Cast(e) => return type_trailing_brace(&e.ty), + Expr::Closure(e) => expr = &e.body, + Expr::Let(e) => expr = &e.expr, + Expr::Macro(e) => return e.mac.delimiter.is_brace(), + Expr::Range(e) => match &e.end { + Some(end) => expr = end, + None => return false, + }, + Expr::RawAddr(e) => expr = &e.expr, + Expr::Reference(e) => expr = &e.expr, + Expr::Return(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Unary(e) => expr = &e.expr, + Expr::Verbatim(e) => return tokens_trailing_brace(e), + Expr::Yield(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + + Expr::Array(_) + | Expr::Await(_) + | Expr::Call(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Repeat(_) + | Expr::Try(_) + | Expr::Tuple(_) => return false, + } + } + + fn type_trailing_brace(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Macro(t) => return t.mac.delimiter.is_brace(), + Type::Path(t) => match last_type_in_path(&t.path) { + Some(t) => ty = t, + None => return false, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Verbatim(t) => return tokens_trailing_brace(t), + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) => return false, + } + } + } + + fn last_type_in_path(path: &Path) -> Option<&Type> { + match &path.segments.last().unwrap().arguments { + PathArguments::None | PathArguments::AngleBracketed(_) => None, + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => None, + ReturnType::Type(_, ret) => Some(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => match last_type_in_path(&t.path) { + Some(t) => ControlFlow::Continue(t), + None => ControlFlow::Break(false), + }, + TypeParamBound::Lifetime(_) | TypeParamBound::PreciseCapture(_) => { + ControlFlow::Break(false) + } + TypeParamBound::Verbatim(t) => ControlFlow::Break(tokens_trailing_brace(t)), + } + } + + fn tokens_trailing_brace(tokens: &TokenStream) -> bool { + if let Some(TokenTree::Group(last)) = tokens.clone().into_iter().last() { + last.delimiter() == Delimiter::Brace + } else { + false + } + } +} diff --git a/rust/syn/custom_keyword.rs b/rust/syn/custom_keyword.rs new file mode 100644 index 000000000..ff2c415f1 --- /dev/null +++ b/rust/syn/custom_keyword.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` or `keyword` and that the resulting parser be invoked +/// with a `kw::` or `keyword::` prefix. +/// +/// ``` +/// mod kw { +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// use syn::{LitBool, LitStr, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// mod kw { +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! custom_keyword { + ($ident:ident) => { + #[allow(non_camel_case_types)] + pub struct $ident { + #[allow(dead_code)] + pub span: $crate::__private::Span, + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( + span: __S, + ) -> $ident { + $ident { + span: $crate::__private::IntoSpans::into_spans(span), + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), + } + } + } + + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => { + // For peek. + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + if let $crate::__private::Some((ident, _rest)) = cursor.ident() { + ident == $crate::__private::stringify!($ident) + } else { + false + } + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let $crate::__private::Some((ident, rest)) = cursor.ident() { + if ident == $crate::__private::stringify!($ident) { + return $crate::__private::Ok(($ident { span: ident.span() }, rest)); + } + } + $crate::__private::Err(cursor.error($crate::__private::concat!( + "expected `", + $crate::__private::stringify!($ident), + "`", + ))) + }) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); + $crate::__private::TokenStreamExt::append(tokens, ident); + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str( + f, + $crate::__private::concat!( + "Keyword [", + $crate::__private::stringify!($ident), + "]", + ), + ) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; +} diff --git a/rust/syn/custom_punctuation.rs b/rust/syn/custom_punctuation.rs new file mode 100644 index 000000000..d37c0e9c3 --- /dev/null +++ b/rust/syn/custom_punctuation.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Define a type that supports parsing and printing a multi-character symbol +/// as if it were a punctuation token. +/// +/// # Usage +/// +/// ``` +/// syn::custom_punctuation!(LeftRightArrow, <=>); +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in punctuation token. +/// +/// - [Peeking] — `input.peek(LeftRightArrow)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #lrarrow ... )` +/// +/// - Construction from a [`Span`] — `let lrarrow = LeftRightArrow(sp)` +/// +/// - Construction from multiple [`Span`] — `let lrarrow = LeftRightArrow([sp, sp, sp])` +/// +/// - Field access to its spans — `let spans = lrarrow.spans` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// ``` +/// use proc_macro2::{TokenStream, TokenTree}; +/// use syn::parse::{Parse, ParseStream, Peek, Result}; +/// use syn::punctuated::Punctuated; +/// use syn::Expr; +/// +/// syn::custom_punctuation!(PathSeparator, ); +/// +/// // expr expr expr ... +/// struct PathSegments { +/// segments: Punctuated, +/// } +/// +/// impl Parse for PathSegments { +/// fn parse(input: ParseStream) -> Result { +/// let mut segments = Punctuated::new(); +/// +/// let first = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(first)?); +/// +/// while input.peek(PathSeparator) { +/// segments.push_punct(input.parse()?); +/// +/// let next = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(next)?); +/// } +/// +/// Ok(PathSegments { segments }) +/// } +/// } +/// +/// fn parse_until(input: ParseStream, end: E) -> Result { +/// let mut tokens = TokenStream::new(); +/// while !input.is_empty() && !input.peek(end) { +/// let next: TokenTree = input.parse()?; +/// tokens.extend(Some(next)); +/// } +/// Ok(tokens) +/// } +/// +/// fn main() { +/// let input = r#" a::b c::d::e "#; +/// let _: PathSegments = syn::parse_str(input).unwrap(); +/// } +/// ``` +#[macro_export] +macro_rules! custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + pub struct $ident { + #[allow(dead_code)] + pub spans: $crate::custom_punctuation_repr!($($tt)+), + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::custom_punctuation_repr!($($tt)+)>>( + spans: __S, + ) -> $ident { + let _validate_len = 0 $(+ $crate::custom_punctuation_len!(strict, $tt))*; + $ident { + spans: $crate::__private::IntoSpans::into_spans(spans) + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) + } + } + + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + let spans: $crate::custom_punctuation_repr!($($tt)+) = + $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; + Ok($ident(spans)) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_repr { + ($($tt:tt)+) => { + [$crate::__private::Span; 0 $(+ $crate::custom_punctuation_len!(lenient, $tt))+] + }; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +#[rustfmt::skip] +macro_rules! custom_punctuation_len { + ($mode:ident, &) => { 1 }; + ($mode:ident, &&) => { 2 }; + ($mode:ident, &=) => { 2 }; + ($mode:ident, @) => { 1 }; + ($mode:ident, ^) => { 1 }; + ($mode:ident, ^=) => { 2 }; + ($mode:ident, :) => { 1 }; + ($mode:ident, ,) => { 1 }; + ($mode:ident, $) => { 1 }; + ($mode:ident, .) => { 1 }; + ($mode:ident, ..) => { 2 }; + ($mode:ident, ...) => { 3 }; + ($mode:ident, ..=) => { 3 }; + ($mode:ident, =) => { 1 }; + ($mode:ident, ==) => { 2 }; + ($mode:ident, =>) => { 2 }; + ($mode:ident, >=) => { 2 }; + ($mode:ident, >) => { 1 }; + ($mode:ident, <-) => { 2 }; + ($mode:ident, <=) => { 2 }; + ($mode:ident, <) => { 1 }; + ($mode:ident, -) => { 1 }; + ($mode:ident, -=) => { 2 }; + ($mode:ident, !=) => { 2 }; + ($mode:ident, !) => { 1 }; + ($mode:ident, |) => { 1 }; + ($mode:ident, |=) => { 2 }; + ($mode:ident, ||) => { 2 }; + ($mode:ident, ::) => { 2 }; + ($mode:ident, %) => { 1 }; + ($mode:ident, %=) => { 2 }; + ($mode:ident, +) => { 1 }; + ($mode:ident, +=) => { 2 }; + ($mode:ident, #) => { 1 }; + ($mode:ident, ?) => { 1 }; + ($mode:ident, ->) => { 2 }; + ($mode:ident, ;) => { 1 }; + ($mode:ident, <<) => { 2 }; + ($mode:ident, <<=) => { 3 }; + ($mode:ident, >>) => { 2 }; + ($mode:ident, >>=) => { 3 }; + ($mode:ident, /) => { 1 }; + ($mode:ident, /=) => { 2 }; + ($mode:ident, *) => { 1 }; + ($mode:ident, *=) => { 2 }; + ($mode:ident, ~) => { 1 }; + (lenient, $tt:tt) => { 0 }; + (strict, $tt:tt) => {{ $crate::custom_punctuation_unexpected!($tt); 0 }}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_unexpected { + () => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! stringify_punct { + ($($tt:tt)+) => { + $crate::__private::concat!($($crate::__private::stringify!($tt)),+) + }; +} diff --git a/rust/syn/data.rs b/rust/syn/data.rs new file mode 100644 index 000000000..c32c1366c --- /dev/null +++ b/rust/syn/data.rs @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +use crate::expr::{Expr, Index, Member}; +use crate::ident::Ident; +use crate::punctuated::{self, Punctuated}; +use crate::restriction::{FieldMutability, Visibility}; +use crate::token; +use crate::ty::Type; + +ast_struct! { + /// An enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Variant { + pub attrs: Vec, + + /// Name of the variant. + pub ident: Ident, + + /// Content stored in the variant. + pub fields: Fields, + + /// Explicit discriminant: `Variant = 1` + pub discriminant: Option<(Token![=], Expr)>, + } +} + +ast_enum_of_structs! { + /// Data stored within an enum variant or struct. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Fields { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + Named(FieldsNamed), + + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + Unnamed(FieldsUnnamed), + + /// Unit struct or unit variant such as `None`. + Unit, + } +} + +ast_struct! { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsNamed { + pub brace_token: token::Brace, + pub named: Punctuated, + } +} + +ast_struct! { + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsUnnamed { + pub paren_token: token::Paren, + pub unnamed: Punctuated, + } +} + +impl Fields { + /// Get an iterator over the borrowed [`Field`] items in this object. This + /// iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + pub fn iter(&self) -> punctuated::Iter { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter(), + Fields::Named(f) => f.named.iter(), + Fields::Unnamed(f) => f.unnamed.iter(), + } + } + + /// Get an iterator over the mutably borrowed [`Field`] items in this + /// object. This iterator can be used to iterate over a named or unnamed + /// struct or variant's fields uniformly. + pub fn iter_mut(&mut self) -> punctuated::IterMut { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter_mut(), + Fields::Named(f) => f.named.iter_mut(), + Fields::Unnamed(f) => f.unnamed.iter_mut(), + } + } + + /// Returns the number of fields. + pub fn len(&self) -> usize { + match self { + Fields::Unit => 0, + Fields::Named(f) => f.named.len(), + Fields::Unnamed(f) => f.unnamed.len(), + } + } + + /// Returns `true` if there are zero fields. + pub fn is_empty(&self) -> bool { + match self { + Fields::Unit => true, + Fields::Named(f) => f.named.is_empty(), + Fields::Unnamed(f) => f.unnamed.is_empty(), + } + } + + return_impl_trait! { + /// Get an iterator over the fields of a struct or variant as [`Member`]s. + /// This iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + /// + /// # Example + /// + /// The following is a simplistic [`Clone`] derive for structs. (A more + /// complete implementation would additionally want to infer trait bounds on + /// the generic type parameters.) + /// + /// ``` + /// # use quote::quote; + /// # + /// fn derive_clone(input: &syn::ItemStruct) -> proc_macro2::TokenStream { + /// let ident = &input.ident; + /// let members = input.fields.members(); + /// let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + /// quote! { + /// impl #impl_generics Clone for #ident #ty_generics #where_clause { + /// fn clone(&self) -> Self { + /// Self { + /// #(#members: self.#members.clone()),* + /// } + /// } + /// } + /// } + /// } + /// ``` + /// + /// For structs with named fields, it produces an expression like `Self { a: + /// self.a.clone() }`. For structs with unnamed fields, `Self { 0: + /// self.0.clone() }`. And for unit structs, `Self {}`. + pub fn members(&self) -> impl Iterator + Clone + '_ [Members] { + Members { + fields: self.iter(), + index: 0, + } + } + } +} + +impl IntoIterator for Fields { + type Item = Field; + type IntoIter = punctuated::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + match self { + Fields::Unit => Punctuated::::new().into_iter(), + Fields::Named(f) => f.named.into_iter(), + Fields::Unnamed(f) => f.unnamed.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Fields { + type Item = &'a Field; + type IntoIter = punctuated::Iter<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut Fields { + type Item = &'a mut Field; + type IntoIter = punctuated::IterMut<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +ast_struct! { + /// A field of a struct or enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Field { + pub attrs: Vec, + + pub vis: Visibility, + + pub mutability: FieldMutability, + + /// Name of the field, if any. + /// + /// Fields of tuple structs have no names. + pub ident: Option, + + pub colon_token: Option, + + pub ty: Type, + } +} + +pub struct Members<'a> { + fields: punctuated::Iter<'a, Field>, + index: u32, +} + +impl<'a> Iterator for Members<'a> { + type Item = Member; + + fn next(&mut self) -> Option { + let field = self.fields.next()?; + let member = match &field.ident { + Some(ident) => Member::Named(ident.clone()), + None => { + #[cfg(all(feature = "parsing", feature = "printing"))] + let span = crate::spanned::Spanned::span(&field.ty); + #[cfg(not(all(feature = "parsing", feature = "printing")))] + let span = proc_macro2::Span::call_site(); + Member::Unnamed(Index { + index: self.index, + span, + }) + } + }; + self.index += 1; + Some(member) + } +} + +impl<'a> Clone for Members<'a> { + fn clone(&self) -> Self { + Members { + fields: self.fields.clone(), + index: self.index, + } + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Field, Fields, FieldsNamed, FieldsUnnamed, Variant}; + use crate::error::Result; + use crate::expr::Expr; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + #[cfg(not(feature = "full"))] + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::restriction::{FieldMutability, Visibility}; + #[cfg(not(feature = "full"))] + use crate::scan_expr::scan_expr; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Variant { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let _visibility: Visibility = input.parse()?; + let ident: Ident = input.parse()?; + let fields = if input.peek(token::Brace) { + Fields::Named(input.parse()?) + } else if input.peek(token::Paren) { + Fields::Unnamed(input.parse()?) + } else { + Fields::Unit + }; + let discriminant = if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + #[cfg(feature = "full")] + let discriminant: Expr = input.parse()?; + #[cfg(not(feature = "full"))] + let discriminant = { + let begin = input.fork(); + let ahead = input.fork(); + let mut discriminant: Result = ahead.parse(); + if discriminant.is_ok() { + input.advance_to(&ahead); + } else if scan_expr(input).is_ok() { + discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); + } + discriminant? + }; + Some((eq_token, discriminant)) + } else { + None + }; + Ok(Variant { + attrs, + ident, + fields, + discriminant, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsNamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsNamed { + brace_token: braced!(content in input), + named: content.parse_terminated(Field::parse_named, Token![,])?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsUnnamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsUnnamed { + paren_token: parenthesized!(content in input), + unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, + }) + } + } + + impl Field { + /// Parses a named (braced struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_named(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); + let ident = if unnamed_field { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let colon_token: Token![:] = input.parse()?; + + let ty: Type = if unnamed_field + && (input.peek(Token![struct]) + || input.peek(Token![union]) && input.peek2(token::Brace)) + { + let begin = input.fork(); + input.call(Ident::parse_any)?; + input.parse::()?; + Type::Verbatim(verbatim::between(&begin, input)) + } else { + input.parse()? + }; + + Ok(Field { + attrs, + vis, + mutability: FieldMutability::None, + ident: Some(ident), + colon_token: Some(colon_token), + ty, + }) + } + + /// Parses an unnamed (tuple struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_unnamed(input: ParseStream) -> Result { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + mutability: FieldMutability::None, + ident: None, + colon_token: None, + ty: input.parse()?, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::data::{Field, FieldsNamed, FieldsUnnamed, Variant}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Variant { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.ident.to_tokens(tokens); + self.fields.to_tokens(tokens); + if let Some((eq_token, disc)) = &self.discriminant { + eq_token.to_tokens(tokens); + disc.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsNamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.named.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsUnnamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.unnamed.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Field { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.vis.to_tokens(tokens); + if let Some(ident) = &self.ident { + ident.to_tokens(tokens); + TokensOrDefault(&self.colon_token).to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } +} diff --git a/rust/syn/derive.rs b/rust/syn/derive.rs new file mode 100644 index 000000000..27fd9f12c --- /dev/null +++ b/rust/syn/derive.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +use crate::data::{Fields, FieldsNamed, Variant}; +use crate::generics::Generics; +use crate::ident::Ident; +use crate::punctuated::Punctuated; +use crate::restriction::Visibility; +use crate::token; + +ast_struct! { + /// Data structure sent to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DeriveInput { + pub attrs: Vec, + pub vis: Visibility, + pub ident: Ident, + pub generics: Generics, + pub data: Data, + } +} + +ast_enum! { + /// The storage of a struct, enum or union data structure. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub enum Data { + Struct(DataStruct), + Enum(DataEnum), + Union(DataUnion), + } +} + +ast_struct! { + /// A struct input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataStruct { + pub struct_token: Token![struct], + pub fields: Fields, + pub semi_token: Option, + } +} + +ast_struct! { + /// An enum input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataEnum { + pub enum_token: Token![enum], + pub brace_token: token::Brace, + pub variants: Punctuated, + } +} + +ast_struct! { + /// An untagged union input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataUnion { + pub union_token: Token![union], + pub fields: FieldsNamed, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Fields, FieldsNamed, Variant}; + use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; + use crate::error::Result; + use crate::generics::{Generics, WhereClause}; + use crate::ident::Ident; + use crate::parse::{Parse, ParseStream}; + use crate::punctuated::Punctuated; + use crate::restriction::Visibility; + use crate::token; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for DeriveInput { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::()?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![struct]) { + let struct_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields, semi) = data_struct(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Struct(DataStruct { + struct_token, + fields, + semi_token: semi, + }), + }) + } else if lookahead.peek(Token![enum]) { + let enum_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, brace, variants) = data_enum(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Enum(DataEnum { + enum_token, + brace_token: brace, + variants, + }), + }) + } else if lookahead.peek(Token![union]) { + let union_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields) = data_union(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Union(DataUnion { + union_token, + fields, + }), + }) + } else { + Err(lookahead.error()) + } + } + } + + pub(crate) fn data_struct( + input: ParseStream, + ) -> Result<(Option, Fields, Option)> { + let mut lookahead = input.lookahead1(); + let mut where_clause = None; + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if where_clause.is_none() && lookahead.peek(token::Paren) { + let fields = input.parse()?; + + lookahead = input.lookahead1(); + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unnamed(fields), Some(semi))) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(token::Brace) { + let fields = input.parse()?; + Ok((where_clause, Fields::Named(fields), None)) + } else if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unit, Some(semi))) + } else { + Err(lookahead.error()) + } + } + + pub(crate) fn data_enum( + input: ParseStream, + ) -> Result<( + Option, + token::Brace, + Punctuated, + )> { + let where_clause = input.parse()?; + + let content; + let brace = braced!(content in input); + let variants = content.parse_terminated(Variant::parse, Token![,])?; + + Ok((where_clause, brace, variants)) + } + + pub(crate) fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + let where_clause = input.parse()?; + let fields = input.parse()?; + Ok((where_clause, fields)) + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::data::Fields; + use crate::derive::{Data, DeriveInput}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for DeriveInput { + fn to_tokens(&self, tokens: &mut TokenStream) { + for attr in self.attrs.outer() { + attr.to_tokens(tokens); + } + self.vis.to_tokens(tokens); + match &self.data { + Data::Struct(d) => d.struct_token.to_tokens(tokens), + Data::Enum(d) => d.enum_token.to_tokens(tokens), + Data::Union(d) => d.union_token.to_tokens(tokens), + } + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.data { + Data::Struct(data) => match &data.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + }, + Data::Enum(data) => { + self.generics.where_clause.to_tokens(tokens); + data.brace_token.surround(tokens, |tokens| { + data.variants.to_tokens(tokens); + }); + } + Data::Union(data) => { + self.generics.where_clause.to_tokens(tokens); + data.fields.to_tokens(tokens); + } + } + } + } +} diff --git a/rust/syn/discouraged.rs b/rust/syn/discouraged.rs new file mode 100644 index 000000000..2acd631ec --- /dev/null +++ b/rust/syn/discouraged.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Extensions to the parsing API with niche applicability. + +use crate::buffer::Cursor; +use crate::error::Result; +use crate::parse::{inner_unexpected, ParseBuffer, Unexpected}; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Delimiter; +use std::cell::Cell; +use std::mem; +use std::rc::Rc; + +/// Extensions to the `ParseStream` API to support speculative parsing. +pub trait Speculative { + /// Advance this parse stream to the position of a forked parse stream. + /// + /// This is the opposite operation to [`ParseStream::fork`]. You can fork a + /// parse stream, perform some speculative parsing, then join the original + /// stream to the fork to "commit" the parsing from the fork to the main + /// stream. + /// + /// If you can avoid doing this, you should, as it limits the ability to + /// generate useful errors. That said, it is often the only way to parse + /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem + /// is that when the fork fails to parse an `A`, it's impossible to tell + /// whether that was because of a syntax error and the user meant to provide + /// an `A`, or that the `A`s are finished and it's time to start parsing + /// `B`s. Use with care. + /// + /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by + /// parsing `B*` and removing the leading members of `A` from the + /// repetition, bypassing the need to involve the downsides associated with + /// speculative parsing. + /// + /// [`ParseStream::fork`]: ParseBuffer::fork + /// + /// # Example + /// + /// There has been chatter about the possibility of making the colons in the + /// turbofish syntax like `path::to::` no longer required by accepting + /// `path::to` in expression position. Specifically, according to [RFC + /// 2544], [`PathSegment`] parsing should always try to consume a following + /// `<` token as the start of generic arguments, and reset to the `<` if + /// that fails (e.g. the token is acting as a less-than operator). + /// + /// This is the exact kind of parsing behavior which requires the "fork, + /// try, commit" behavior that [`ParseStream::fork`] discourages. With + /// `advance_to`, we can avoid having to parse the speculatively parsed + /// content a second time. + /// + /// This change in behavior can be implemented in syn by replacing just the + /// `Parse` implementation for `PathSegment`: + /// + /// ``` + /// # use syn::ext::IdentExt; + /// use syn::parse::discouraged::Speculative; + /// # use syn::parse::{Parse, ParseStream}; + /// # use syn::{Ident, PathArguments, Result, Token}; + /// + /// pub struct PathSegment { + /// pub ident: Ident, + /// pub arguments: PathArguments, + /// } + /// # + /// # impl From for PathSegment + /// # where + /// # T: Into, + /// # { + /// # fn from(ident: T) -> Self { + /// # PathSegment { + /// # ident: ident.into(), + /// # arguments: PathArguments::None, + /// # } + /// # } + /// # } + /// + /// impl Parse for PathSegment { + /// fn parse(input: ParseStream) -> Result { + /// if input.peek(Token![super]) + /// || input.peek(Token![self]) + /// || input.peek(Token![Self]) + /// || input.peek(Token![crate]) + /// { + /// let ident = input.call(Ident::parse_any)?; + /// return Ok(PathSegment::from(ident)); + /// } + /// + /// let ident = input.parse()?; + /// if input.peek(Token![::]) && input.peek3(Token![<]) { + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(input.parse()?), + /// }); + /// } + /// if input.peek(Token![<]) && !input.peek(Token![<=]) { + /// let fork = input.fork(); + /// if let Ok(arguments) = fork.parse() { + /// input.advance_to(&fork); + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(arguments), + /// }); + /// } + /// } + /// Ok(PathSegment::from(ident)) + /// } + /// } + /// + /// # syn::parse_str::("a").unwrap(); + /// ``` + /// + /// # Drawbacks + /// + /// The main drawback of this style of speculative parsing is in error + /// presentation. Even if the lookahead is the "correct" parse, the error + /// that is shown is that of the "fallback" parse. To use the same example + /// as the turbofish above, take the following unfinished "turbofish": + /// + /// ```text + /// let _ = f<&'a fn(), for<'a> serde::>(); + /// ``` + /// + /// If this is parsed as generic arguments, we can provide the error message + /// + /// ```text + /// error: expected identifier + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^ + /// ``` + /// + /// but if parsed using the above speculative parsing, it falls back to + /// assuming that the `<` is a less-than when it fails to parse the generic + /// arguments, and tries to interpret the `&'a` as the start of a labelled + /// loop, resulting in the much less helpful error + /// + /// ```text + /// error: expected `:` + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^^ + /// ``` + /// + /// This can be mitigated with various heuristics (two examples: show both + /// forks' parse errors, or show the one that consumed more tokens), but + /// when you can control the grammar, sticking to something that can be + /// parsed LL(3) and without the LL(*) speculative parsing this makes + /// possible, displaying reasonable errors becomes much more simple. + /// + /// [RFC 2544]: https://github.com/rust-lang/rfcs/pull/2544 + /// [`PathSegment`]: crate::PathSegment + /// + /// # Performance + /// + /// This method performs a cheap fixed amount of work that does not depend + /// on how far apart the two streams are positioned. + /// + /// # Panics + /// + /// The forked stream in the argument of `advance_to` must have been + /// obtained by forking `self`. Attempting to advance to any other stream + /// will cause a panic. + fn advance_to(&self, fork: &Self); +} + +impl<'a> Speculative for ParseBuffer<'a> { + fn advance_to(&self, fork: &Self) { + if !crate::buffer::same_scope(self.cursor(), fork.cursor()) { + panic!("fork was not derived from the advancing parse stream"); + } + + let (self_unexp, self_sp) = inner_unexpected(self); + let (fork_unexp, fork_sp) = inner_unexpected(fork); + if !Rc::ptr_eq(&self_unexp, &fork_unexp) { + match (fork_sp, self_sp) { + // Unexpected set on the fork, but not on `self`, copy it over. + (Some((span, delimiter)), None) => { + self_unexp.set(Unexpected::Some(span, delimiter)); + } + // Unexpected unset. Use chain to propagate errors from fork. + (None, None) => { + fork_unexp.set(Unexpected::Chain(self_unexp)); + + // Ensure toplevel 'unexpected' tokens from the fork don't + // propagate up the chain by replacing the root `unexpected` + // pointer, only 'unexpected' tokens from existing group + // parsers should propagate. + fork.unexpected + .set(Some(Rc::new(Cell::new(Unexpected::None)))); + } + // Unexpected has been set on `self`. No changes needed. + (_, Some(_)) => {} + } + } + + // See comment on `cell` in the struct definition. + self.cell + .set(unsafe { mem::transmute::>(fork.cursor()) }); + } +} + +/// Extensions to the `ParseStream` API to support manipulating invisible +/// delimiters the same as if they were visible. +pub trait AnyDelimiter { + /// Returns the delimiter, the span of the delimiter token, and the nested + /// contents for further parsing. + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; +} + +impl<'a> AnyDelimiter for ParseBuffer<'a> { + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { + self.step(|cursor| { + if let Some((content, delimiter, span, rest)) = cursor.any_group() { + let scope = span.close(); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(self); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((delimiter, span, content), rest)) + } else { + Err(cursor.error("expected any delimiter")) + } + }) + } +} diff --git a/rust/syn/drops.rs b/rust/syn/drops.rs new file mode 100644 index 000000000..2c4ae333d --- /dev/null +++ b/rust/syn/drops.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::iter; +use std::mem::ManuallyDrop; +use std::ops::{Deref, DerefMut}; +use std::option; +use std::slice; + +#[repr(transparent)] +pub(crate) struct NoDrop(ManuallyDrop); + +impl NoDrop { + pub(crate) fn new(value: T) -> Self + where + T: TrivialDrop, + { + NoDrop(ManuallyDrop::new(value)) + } +} + +impl Deref for NoDrop { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for NoDrop { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +pub(crate) trait TrivialDrop {} + +impl TrivialDrop for iter::Empty {} +impl TrivialDrop for slice::Iter<'_, T> {} +impl TrivialDrop for slice::IterMut<'_, T> {} +impl TrivialDrop for option::IntoIter<&T> {} +impl TrivialDrop for option::IntoIter<&mut T> {} + +#[test] +fn test_needs_drop() { + use std::mem::needs_drop; + + struct NeedsDrop; + + impl Drop for NeedsDrop { + fn drop(&mut self) {} + } + + assert!(needs_drop::()); + + // Test each of the types with a handwritten TrivialDrop impl above. + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); +} diff --git a/rust/syn/error.rs b/rust/syn/error.rs new file mode 100644 index 000000000..6fa0faf7f --- /dev/null +++ b/rust/syn/error.rs @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +use crate::thread::ThreadBound; +use proc_macro2::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; +#[cfg(feature = "printing")] +use quote::ToTokens; +use std::fmt::{self, Debug, Display}; +use std::slice; +use std::vec; + +/// The result of a Syn parser. +pub type Result = std::result::Result; + +/// Error returned when a Syn parser cannot parse the input tokens. +/// +/// # Error reporting in proc macros +/// +/// The correct way to report errors back to the compiler from a procedural +/// macro is by emitting an appropriately spanned invocation of +/// [`compile_error!`] in the generated code. This produces a better diagnostic +/// message than simply panicking the macro. +/// +/// [`compile_error!`]: std::compile_error! +/// +/// When parsing macro input, the [`parse_macro_input!`] macro handles the +/// conversion to `compile_error!` automatically. +/// +/// [`parse_macro_input!`]: crate::parse_macro_input! +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::parse::{Parse, ParseStream, Result}; +/// use syn::{parse_macro_input, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as MyAttrArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// +/// struct MyAttrArgs { +/// # _k: [(); { stringify! { +/// ... +/// # }; 0 }] +/// } +/// +/// impl Parse for MyAttrArgs { +/// fn parse(input: ParseStream) -> Result { +/// # stringify! { +/// ... +/// # }; +/// # unimplemented!() +/// } +/// } +/// ``` +/// +/// For errors that arise later than the initial parsing stage, the +/// [`.to_compile_error()`] or [`.into_compile_error()`] methods can be used to +/// perform an explicit conversion to `compile_error!`. +/// +/// [`.to_compile_error()`]: Error::to_compile_error +/// [`.into_compile_error()`]: Error::into_compile_error +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, DeriveInput}; +/// # +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_derive(MyDerive)] +/// # }; +/// pub fn my_derive(input: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(input as DeriveInput); +/// +/// // fn(DeriveInput) -> syn::Result +/// expand::my_derive(input) +/// .unwrap_or_else(syn::Error::into_compile_error) +/// .into() +/// } +/// # +/// # mod expand { +/// # use proc_macro2::TokenStream; +/// # use syn::{DeriveInput, Result}; +/// # +/// # pub fn my_derive(input: DeriveInput) -> Result { +/// # unimplemented!() +/// # } +/// # } +/// ``` +pub struct Error { + messages: Vec, +} + +struct ErrorMessage { + // Span is implemented as an index into a thread-local interner to keep the + // size small. It is not safe to access from a different thread. We want + // errors to be Send and Sync to play nicely with ecosystem crates for error + // handling, so pin the span we're given to its original thread and assume + // it is Span::call_site if accessed from any other thread. + span: ThreadBound, + message: String, +} + +// Cannot use std::ops::Range because that does not implement Copy, +// whereas ThreadBound requires a Copy impl as a way to ensure no Drop impls +// are involved. +struct SpanRange { + start: Span, + end: Span, +} + +#[cfg(test)] +struct _Test +where + Error: Send + Sync; + +impl Error { + /// Usually the [`ParseStream::error`] method will be used instead, which + /// automatically uses the correct span from the current position of the + /// parse stream. + /// + /// Use `Error::new` when the error needs to be triggered on some span other + /// than where the parse stream is currently positioned. + /// + /// [`ParseStream::error`]: crate::parse::ParseBuffer::error + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, LitStr, Result, Token}; + /// use syn::parse::ParseStream; + /// + /// // Parses input that looks like `name = "string"` where the key must be + /// // the identifier `name` and the value may be any string literal. + /// // Returns the string literal. + /// fn parse_name(input: ParseStream) -> Result { + /// let name_token: Ident = input.parse()?; + /// if name_token != "name" { + /// // Trigger an error not on the current position of the stream, + /// // but on the position of the unexpected identifier. + /// return Err(Error::new(name_token.span(), "expected `name`")); + /// } + /// input.parse::()?; + /// let s: LitStr = input.parse()?; + /// Ok(s) + /// } + /// ``` + pub fn new(span: Span, message: T) -> Self { + return new(span, message.to_string()); + + fn new(span: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { + start: span, + end: span, + }), + message, + }], + } + } + } + + /// Creates an error with the specified message spanning the given syntax + /// tree node. + /// + /// Unlike the `Error::new` constructor, this constructor takes an argument + /// `tokens` which is a syntax tree node. This allows the resulting `Error` + /// to attempt to span all tokens inside of `tokens`. While you would + /// typically be able to use the `Spanned` trait with the above `Error::new` + /// constructor, implementation limitations today mean that + /// `Error::new_spanned` may provide a higher-quality error message on + /// stable Rust. + /// + /// When in doubt it's recommended to stick to `Error::new` (or + /// `ParseStream::error`)! + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn new_spanned(tokens: T, message: U) -> Self { + return new_spanned(tokens.into_token_stream(), message.to_string()); + + fn new_spanned(tokens: TokenStream, message: String) -> Error { + let mut iter = tokens.into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } + } + + /// The source location of the error. + /// + /// Spans are not thread-safe so this function returns `Span::call_site()` + /// if called from a different thread than the one on which the `Error` was + /// originally created. + pub fn span(&self) -> Span { + let SpanRange { start, end } = match self.messages[0].span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + start.join(end).unwrap_or(start) + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// The [`parse_macro_input!`] macro provides a convenient way to invoke + /// this method correctly in a procedural macro. + /// + /// [`compile_error!`]: std::compile_error! + /// [`parse_macro_input!`]: crate::parse_macro_input! + pub fn to_compile_error(&self) -> TokenStream { + self.messages + .iter() + .map(ErrorMessage::to_compile_error) + .collect() + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// [`compile_error!`]: std::compile_error! + /// + /// # Example + /// + /// ``` + /// # extern crate proc_macro; + /// # + /// use proc_macro::TokenStream; + /// use syn::{parse_macro_input, DeriveInput, Error}; + /// + /// # const _: &str = stringify! { + /// #[proc_macro_derive(MyTrait)] + /// # }; + /// pub fn derive_my_trait(input: TokenStream) -> TokenStream { + /// let input = parse_macro_input!(input as DeriveInput); + /// my_trait::expand(input) + /// .unwrap_or_else(Error::into_compile_error) + /// .into() + /// } + /// + /// mod my_trait { + /// use proc_macro2::TokenStream; + /// use syn::{DeriveInput, Result}; + /// + /// pub(crate) fn expand(input: DeriveInput) -> Result { + /// /* ... */ + /// # unimplemented!() + /// } + /// } + /// ``` + pub fn into_compile_error(self) -> TokenStream { + self.to_compile_error() + } + + /// Add another error message to self such that when `to_compile_error()` is + /// called, both errors will be emitted together. + pub fn combine(&mut self, another: Error) { + self.messages.extend(another.messages); + } +} + +impl ErrorMessage { + fn to_compile_error(&self) -> TokenStream { + let (start, end) = match self.span.get() { + Some(range) => (range.start, range.end), + None => (Span::call_site(), Span::call_site()), + }; + + // ::core::compile_error!($message) + TokenStream::from_iter([ + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("core", start)), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("compile_error", start)), + TokenTree::Punct({ + let mut punct = Punct::new('!', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Group({ + let mut group = Group::new(Delimiter::Brace, { + TokenStream::from_iter([TokenTree::Literal({ + let mut string = Literal::string(&self.message); + string.set_span(end); + string + })]) + }); + group.set_span(end); + group + }), + ]) + } +} + +#[cfg(feature = "parsing")] +pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { + if cursor.eof() { + Error::new(scope, format!("unexpected end of input, {}", message)) + } else { + let span = crate::buffer::open_span_of_group(cursor); + Error::new(span, message) + } +} + +#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] +pub(crate) fn new2(start: Span, end: Span, message: T) -> Error { + return new2(start, end, message.to_string()); + + fn new2(start: Span, end: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } +} + +impl Debug for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.messages.len() == 1 { + formatter + .debug_tuple("Error") + .field(&self.messages[0]) + .finish() + } else { + formatter + .debug_tuple("Error") + .field(&self.messages) + .finish() + } + } +} + +impl Debug for ErrorMessage { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.message, formatter) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(&self.messages[0].message) + } +} + +impl Clone for Error { + fn clone(&self) -> Self { + Error { + messages: self.messages.clone(), + } + } +} + +impl Clone for ErrorMessage { + fn clone(&self) -> Self { + ErrorMessage { + span: self.span, + message: self.message.clone(), + } + } +} + +impl Clone for SpanRange { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SpanRange {} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: LexError) -> Self { + Error::new(err.span(), err) + } +} + +impl IntoIterator for Error { + type Item = Error; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + messages: self.messages.into_iter(), + } + } +} + +pub struct IntoIter { + messages: vec::IntoIter, +} + +impl Iterator for IntoIter { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?], + }) + } +} + +impl<'a> IntoIterator for &'a Error { + type Item = Error; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Iter { + messages: self.messages.iter(), + } + } +} + +pub struct Iter<'a> { + messages: slice::Iter<'a, ErrorMessage>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?.clone()], + }) + } +} + +impl Extend for Error { + fn extend>(&mut self, iter: T) { + for err in iter { + self.combine(err); + } + } +} diff --git a/rust/syn/export.rs b/rust/syn/export.rs new file mode 100644 index 000000000..05dad324b --- /dev/null +++ b/rust/syn/export.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[doc(hidden)] +pub use std::clone::Clone; +#[doc(hidden)] +pub use std::cmp::{Eq, PartialEq}; +#[doc(hidden)] +pub use std::concat; +#[doc(hidden)] +pub use std::default::Default; +#[doc(hidden)] +pub use std::fmt::Debug; +#[doc(hidden)] +pub use std::hash::{Hash, Hasher}; +#[doc(hidden)] +pub use std::marker::Copy; +#[doc(hidden)] +pub use std::option::Option::{None, Some}; +#[doc(hidden)] +pub use std::result::Result::{Err, Ok}; +#[doc(hidden)] +pub use std::stringify; + +#[doc(hidden)] +pub type Formatter<'a> = std::fmt::Formatter<'a>; +#[doc(hidden)] +pub type FmtResult = std::fmt::Result; + +#[doc(hidden)] +pub type bool = std::primitive::bool; +#[doc(hidden)] +pub type str = std::primitive::str; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote; + +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream2 = proc_macro2::TokenStream; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::group::{parse_braces, parse_brackets, parse_parens}; + +#[doc(hidden)] +pub use crate::span::IntoSpans; + +#[cfg(all(feature = "parsing", feature = "printing"))] +#[doc(hidden)] +pub use crate::parse_quote::parse as parse_quote; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::parsing::{peek_punct, punct as parse_punct}; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use crate::token::printing::punct as print_punct; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::private::CustomToken; + +#[cfg(feature = "proc-macro")] +#[doc(hidden)] +pub type TokenStream = proc_macro::TokenStream; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote::{ToTokens, TokenStreamExt}; + +#[doc(hidden)] +pub struct private(pub(crate) ()); diff --git a/rust/syn/expr.rs b/rust/syn/expr.rs new file mode 100644 index 000000000..deb918436 --- /dev/null +++ b/rust/syn/expr.rs @@ -0,0 +1,4175 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +#[cfg(all(feature = "parsing", feature = "full"))] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::ext::IdentExt as _; +#[cfg(feature = "full")] +use crate::generics::BoundLifetimes; +use crate::ident::Ident; +#[cfg(any(feature = "parsing", feature = "full"))] +use crate::lifetime::Lifetime; +use crate::lit::Lit; +use crate::mac::Macro; +use crate::op::{BinOp, UnOp}; +#[cfg(feature = "parsing")] +use crate::parse::ParseStream; +#[cfg(feature = "full")] +use crate::pat::Pat; +use crate::path::{AngleBracketedGenericArguments, Path, QSelf}; +use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +use crate::stmt::Block; +use crate::token; +#[cfg(feature = "full")] +use crate::ty::ReturnType; +use crate::ty::Type; +use proc_macro2::{Span, TokenStream}; +#[cfg(feature = "printing")] +use quote::IdentFragment; +#[cfg(feature = "printing")] +use std::fmt::{self, Display}; +use std::hash::{Hash, Hasher}; +#[cfg(all(feature = "parsing", feature = "full"))] +use std::mem; + +ast_enum_of_structs! { + /// A Rust expression. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature, but most of the variants are not available unless "full" is enabled.* + /// + /// # Syntax tree enums + /// + /// This type is a syntax tree enum. In Syn this and other syntax tree enums + /// are designed to be traversed using the following rebinding idiom. + /// + /// ``` + /// # use syn::Expr; + /// # + /// # fn example(expr: Expr) { + /// # const IGNORE: &str = stringify! { + /// let expr: Expr = /* ... */; + /// # }; + /// match expr { + /// Expr::MethodCall(expr) => { + /// /* ... */ + /// } + /// Expr::Cast(expr) => { + /// /* ... */ + /// } + /// Expr::If(expr) => { + /// /* ... */ + /// } + /// + /// /* ... */ + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// We begin with a variable `expr` of type `Expr` that has no fields + /// (because it is an enum), and by matching on it and rebinding a variable + /// with the same name `expr` we effectively imbue our variable with all of + /// the data fields provided by the variant that it turned out to be. So for + /// example above if we ended up in the `MethodCall` case then we get to use + /// `expr.receiver`, `expr.args` etc; if we ended up in the `If` case we get + /// to use `expr.cond`, `expr.then_branch`, `expr.else_branch`. + /// + /// This approach avoids repeating the variant names twice on every line. + /// + /// ``` + /// # use syn::{Expr, ExprMethodCall}; + /// # + /// # fn example(expr: Expr) { + /// // Repetitive; recommend not doing this. + /// match expr { + /// Expr::MethodCall(ExprMethodCall { method, args, .. }) => { + /// # } + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// In general, the name to which a syntax tree enum variant is bound should + /// be a suitable name for the complete syntax tree enum type. + /// + /// ``` + /// # use syn::{Expr, ExprField}; + /// # + /// # fn example(discriminant: ExprField) { + /// // Binding is called `base` which is the name I would use if I were + /// // assigning `*discriminant.base` without an `if let`. + /// if let Expr::Tuple(base) = *discriminant.base { + /// # } + /// # } + /// ``` + /// + /// A sign that you may not be choosing the right variable names is if you + /// see names getting repeated in your code, like accessing + /// `receiver.receiver` or `pat.pat` or `cond.cond`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum Expr { + /// A slice literal expression: `[a, b, c, d]`. + Array(ExprArray), + + /// An assignment expression: `a = compute()`. + Assign(ExprAssign), + + /// An async block: `async { ... }`. + Async(ExprAsync), + + /// An await expression: `fut.await`. + Await(ExprAwait), + + /// A binary operation: `a + b`, `a += b`. + Binary(ExprBinary), + + /// A blocked scope: `{ ... }`. + Block(ExprBlock), + + /// A `break`, with an optional label to break and an optional + /// expression. + Break(ExprBreak), + + /// A function call expression: `invoke(a, b)`. + Call(ExprCall), + + /// A cast expression: `foo as f64`. + Cast(ExprCast), + + /// A closure expression: `|a, b| a + b`. + Closure(ExprClosure), + + /// A const block: `const { ... }`. + Const(ExprConst), + + /// A `continue`, with an optional label. + Continue(ExprContinue), + + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + Field(ExprField), + + /// A for loop: `for pat in expr { ... }`. + ForLoop(ExprForLoop), + + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + Group(ExprGroup), + + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + If(ExprIf), + + /// A square bracketed indexing expression: `vector[2]`. + Index(ExprIndex), + + /// The inferred value of a const generic argument, denoted `_`. + Infer(ExprInfer), + + /// A `let` guard: `let Some(x) = opt`. + Let(ExprLet), + + /// A literal in place of an expression: `1`, `"foo"`. + Lit(ExprLit), + + /// Conditionless loop: `loop { ... }`. + Loop(ExprLoop), + + /// A macro invocation expression: `format!("{}", q)`. + Macro(ExprMacro), + + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + Match(ExprMatch), + + /// A method call expression: `x.foo::(a, b)`. + MethodCall(ExprMethodCall), + + /// A parenthesized expression: `(a + b)`. + Paren(ExprParen), + + /// A path like `std::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + Path(ExprPath), + + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + Range(ExprRange), + + /// Address-of operation: `&raw const place` or `&raw mut place`. + RawAddr(ExprRawAddr), + + /// A referencing operation: `&a` or `&mut a`. + Reference(ExprReference), + + /// An array literal constructed from one repeated element: `[0u8; N]`. + Repeat(ExprRepeat), + + /// A `return`, with an optional value to be returned. + Return(ExprReturn), + + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + Struct(ExprStruct), + + /// A try-expression: `expr?`. + Try(ExprTry), + + /// A try block: `try { ... }`. + TryBlock(ExprTryBlock), + + /// A tuple expression: `(a, b, c, d)`. + Tuple(ExprTuple), + + /// A unary operation: `!x`, `*x`. + Unary(ExprUnary), + + /// An unsafe block: `unsafe { ... }`. + Unsafe(ExprUnsafe), + + /// Tokens in expression position not interpreted by Syn. + Verbatim(TokenStream), + + /// A while loop: `while expr { ... }`. + While(ExprWhile), + + /// A yield expression: `yield expr`. + Yield(ExprYield), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match expr { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Expr::Array(expr) => {...} + // Expr::Assign(expr) => {...} + // ... + // Expr::Yield(expr) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A slice literal expression: `[a, b, c, d]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprArray #full { + pub attrs: Vec, + pub bracket_token: token::Bracket, + pub elems: Punctuated, + } +} + +ast_struct! { + /// An assignment expression: `a = compute()`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAssign #full { + pub attrs: Vec, + pub left: Box, + pub eq_token: Token![=], + pub right: Box, + } +} + +ast_struct! { + /// An async block: `async { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAsync #full { + pub attrs: Vec, + pub async_token: Token![async], + pub capture: Option, + pub block: Block, + } +} + +ast_struct! { + /// An await expression: `fut.await`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAwait #full { + pub attrs: Vec, + pub base: Box, + pub dot_token: Token![.], + pub await_token: Token![await], + } +} + +ast_struct! { + /// A binary operation: `a + b`, `a += b`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprBinary { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + +ast_struct! { + /// A blocked scope: `{ ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprBlock #full { + pub attrs: Vec, + pub label: Option