From 33a154a96e71a34a1bcca9f40da343dbbf7b38b4 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 1 Oct 2026 14:52:55 +0000 Subject: selftests/bpf: Test packet range of pointers sharing an id Add tests where two packet pointers share an id and tightening one pointer's umax from its var_off would put it less than their constant distance from the other's umax: with an index & 0x38 capped at 50, the base pointer keeps umax 50, so the pointer 8 bytes further on must keep umax 58, even though its known bits allow at most 56. These refused a valid program or accepted an out-of-bounds access before the fix: - check the advanced copy, load through the base: valid, was refused; - check the base, load the byte at base + 1 through a copy advanced by 8: was accepted; - check base + 4, load 4 bytes at base + 2 through base + 8: reads two bytes past the checked range, was accepted; - the same as the second with data_meta pointers checked against data: was accepted. These pass with and without the fix and cover nearby paths: - subtract an unknown scalar from a checked pointer and load below it (the range is kept across a new id); - reach a load through two paths whose checks cover 8 and 7 bytes after the loaded pointer; the second path must not be pruned by the first; - spill a copy of a pointer, check the pointer, fill the copy and load one byte past the checked range: the load is refused, and the copy has the range of the check. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261001145255.855630-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- rust/.gitignore | 10 + rust/.kunitconfig | 3 + rust/Makefile | 815 +++ rust/bindgen_parameters | 73 + rust/bindings/bindings_helper.h | 172 + rust/bindings/lib.rs | 85 + rust/build_error.rs | 31 + rust/compiler_builtins.rs | 107 + rust/exports.c | 29 + rust/ffi.rs | 50 + rust/helpers/atomic.c | 1035 +++ rust/helpers/atomic_ext.c | 87 + rust/helpers/auxiliary.c | 15 + rust/helpers/barrier.c | 48 + rust/helpers/binder.c | 27 + rust/helpers/bitmap.c | 10 + rust/helpers/bitops.c | 69 + rust/helpers/blk.c | 14 + rust/helpers/bug.c | 13 + rust/helpers/build_assert.c | 25 + rust/helpers/build_bug.c | 8 + rust/helpers/clk.c | 68 + rust/helpers/completion.c | 8 + rust/helpers/cpu.c | 8 + rust/helpers/cpufreq.c | 11 + rust/helpers/cpumask.c | 83 + rust/helpers/cred.c | 13 + rust/helpers/device.c | 32 + rust/helpers/dma-resv.c | 14 + rust/helpers/dma.c | 51 + rust/helpers/drm.c | 78 + rust/helpers/drm_gpuvm.c | 26 + rust/helpers/err.c | 18 + rust/helpers/fs.c | 12 + rust/helpers/fwctl.c | 17 + rust/helpers/gpu.c | 17 + rust/helpers/helpers.c | 108 + rust/helpers/interrupt.c | 13 + rust/helpers/io.c | 161 + rust/helpers/irq.c | 11 + rust/helpers/jump_label.c | 14 + rust/helpers/kunit.c | 8 + rust/helpers/list.c | 17 + rust/helpers/maple_tree.c | 9 + rust/helpers/mm.c | 50 + rust/helpers/mutex.c | 30 + rust/helpers/net/genetlink.c | 46 + rust/helpers/of.c | 8 + rust/helpers/page.c | 28 + rust/helpers/pci.c | 59 + rust/helpers/pid_namespace.c | 28 + rust/helpers/platform.c | 8 + rust/helpers/poll.c | 11 + rust/helpers/processor.c | 8 + rust/helpers/property.c | 8 + rust/helpers/pwm.c | 20 + rust/helpers/rbtree.c | 20 + rust/helpers/rcu.c | 13 + rust/helpers/refcount.c | 28 + rust/helpers/regulator.c | 57 + rust/helpers/scatterlist.c | 26 + rust/helpers/security.c | 48 + rust/helpers/serdev.c | 22 + rust/helpers/signal.c | 8 + rust/helpers/slab.c | 17 + rust/helpers/spinlock.c | 53 + rust/helpers/srcu.c | 35 + rust/helpers/string.c | 8 + rust/helpers/sync.c | 18 + rust/helpers/task.c | 57 + rust/helpers/time.c | 40 + rust/helpers/uaccess.c | 29 + rust/helpers/usb.c | 9 + rust/helpers/vmalloc.c | 16 + rust/helpers/wait.c | 8 + rust/helpers/workqueue.c | 16 + rust/helpers/xarray.c | 28 + rust/kernel/.gitignore | 5 + rust/kernel/Kconfig.test | 86 + rust/kernel/acpi.rs | 55 + rust/kernel/alloc.rs | 273 + rust/kernel/alloc/allocator.rs | 323 + rust/kernel/alloc/allocator/iter.rs | 100 + rust/kernel/alloc/kbox.rs | 758 ++ rust/kernel/alloc/kvec.rs | 1688 +++++ rust/kernel/alloc/kvec/errors.rs | 66 + rust/kernel/alloc/layout.rs | 121 + rust/kernel/auxiliary.rs | 587 ++ rust/kernel/bitfield.rs | 865 +++ rust/kernel/bitmap.rs | 620 ++ rust/kernel/bits.rs | 205 + rust/kernel/block.rs | 18 + rust/kernel/block/mq.rs | 103 + rust/kernel/block/mq/gen_disk.rs | 241 + rust/kernel/block/mq/operations.rs | 286 + rust/kernel/block/mq/request.rs | 257 + rust/kernel/block/mq/tag_set.rs | 85 + rust/kernel/bug.rs | 154 + rust/kernel/build_assert.rs | 219 + rust/kernel/clk.rs | 335 + rust/kernel/configfs.rs | 1047 +++ rust/kernel/cpu.rs | 152 + rust/kernel/cpufreq.rs | 1416 ++++ rust/kernel/cpumask.rs | 343 + rust/kernel/cred.rs | 92 + rust/kernel/debugfs.rs | 724 ++ rust/kernel/debugfs/callback_adapters.rs | 130 + rust/kernel/debugfs/entry.rs | 170 + rust/kernel/debugfs/file_ops.rs | 388 + rust/kernel/debugfs/traits.rs | 321 + rust/kernel/device.rs | 952 +++ rust/kernel/device/property.rs | 633 ++ rust/kernel/device_id.rs | 218 + rust/kernel/devres.rs | 557 ++ rust/kernel/dma.rs | 1251 ++++ rust/kernel/driver.rs | 326 + rust/kernel/drm/device.rs | 588 ++ rust/kernel/drm/driver.rs | 226 + rust/kernel/drm/file.rs | 103 + rust/kernel/drm/gem/mod.rs | 405 ++ rust/kernel/drm/gem/shmem.rs | 721 ++ rust/kernel/drm/gpuvm/mod.rs | 334 + rust/kernel/drm/gpuvm/sm_ops.rs | 429 ++ rust/kernel/drm/gpuvm/va.rs | 176 + rust/kernel/drm/gpuvm/vm_bo.rs | 258 + rust/kernel/drm/ioctl.rs | 217 + rust/kernel/drm/mod.rs | 26 + rust/kernel/error.rs | 663 ++ rust/kernel/faux.rs | 143 + rust/kernel/firmware.rs | 383 + rust/kernel/fmt.rs | 298 + rust/kernel/fs.rs | 11 + rust/kernel/fs/file.rs | 473 ++ rust/kernel/fs/kiocb.rs | 68 + rust/kernel/fwctl.rs | 593 ++ rust/kernel/generated_arch_reachable_asm.rs.S | 7 + rust/kernel/generated_arch_static_branch_asm.rs.S | 7 + rust/kernel/generated_arch_warn_asm.rs.S | 7 + rust/kernel/gpu.rs | 6 + rust/kernel/gpu/buddy.rs | 614 ++ rust/kernel/i2c.rs | 596 ++ rust/kernel/id_pool.rs | 295 + rust/kernel/impl_flags.rs | 275 + rust/kernel/init.rs | 284 + rust/kernel/interop.rs | 9 + rust/kernel/interop/list.rs | 339 + rust/kernel/interrupt.rs | 89 + rust/kernel/io.rs | 1754 +++++ rust/kernel/io/mem.rs | 329 + rust/kernel/io/poll.rs | 183 + rust/kernel/io/register.rs | 1027 +++ rust/kernel/io/resource.rs | 235 + rust/kernel/ioctl.rs | 72 + rust/kernel/iommu/mod.rs | 5 + rust/kernel/iommu/pgtable.rs | 264 + rust/kernel/iov.rs | 314 + rust/kernel/irq.rs | 24 + rust/kernel/irq/flags.rs | 126 + rust/kernel/irq/request.rs | 483 ++ rust/kernel/jump_label.rs | 86 + rust/kernel/kunit.rs | 354 + rust/kernel/lib.rs | 312 + rust/kernel/list.rs | 1222 ++++ rust/kernel/list/arc.rs | 500 ++ rust/kernel/list/arc_field.rs | 97 + rust/kernel/list/impl_list_item_mod.rs | 361 + rust/kernel/maple_tree.rs | 656 ++ rust/kernel/miscdevice.rs | 441 ++ rust/kernel/mm.rs | 297 + rust/kernel/mm/mmput_async.rs | 68 + rust/kernel/mm/virt.rs | 470 ++ rust/kernel/module.rs | 80 + rust/kernel/module_param.rs | 204 + rust/kernel/net/mod.rs | 8 + rust/kernel/net/netlink.rs | 337 + rust/kernel/net/phy.rs | 851 +++ rust/kernel/net/phy/reg.rs | 226 + rust/kernel/num.rs | 88 + rust/kernel/num/bounded.rs | 1165 +++ rust/kernel/num/casts.rs | 298 + rust/kernel/of.rs | 55 + rust/kernel/opp.rs | 1148 +++ rust/kernel/page.rs | 382 + rust/kernel/pci.rs | 545 ++ rust/kernel/pci/id.rs | 580 ++ rust/kernel/pci/io.rs | 348 + rust/kernel/pci/irq.rs | 236 + rust/kernel/pid_namespace.rs | 65 + rust/kernel/platform.rs | 589 ++ rust/kernel/prelude.rs | 126 + rust/kernel/print.rs | 578 ++ rust/kernel/processor.rs | 14 + rust/kernel/ptr.rs | 295 + rust/kernel/ptr/projection.rs | 366 + rust/kernel/pwm.rs | 739 ++ rust/kernel/rbtree.rs | 1433 ++++ rust/kernel/regulator.rs | 398 + rust/kernel/revocable.rs | 271 + rust/kernel/safety.rs | 53 + rust/kernel/scatterlist.rs | 492 ++ rust/kernel/security.rs | 111 + rust/kernel/seq_file.rs | 53 + rust/kernel/serdev.rs | 604 ++ rust/kernel/sizes.rs | 134 + rust/kernel/soc.rs | 135 + rust/kernel/std_vendor.rs | 164 + rust/kernel/str.rs | 918 +++ rust/kernel/str/parse_int.rs | 148 + rust/kernel/sync.rs | 171 + rust/kernel/sync/arc.rs | 918 +++ rust/kernel/sync/arc/std_vendor.rs | 30 + rust/kernel/sync/aref.rs | 249 + rust/kernel/sync/atomic.rs | 850 +++ rust/kernel/sync/atomic/internal.rs | 352 + rust/kernel/sync/atomic/ordering.rs | 104 + rust/kernel/sync/atomic/predefine.rs | 335 + rust/kernel/sync/barrier.rs | 132 + rust/kernel/sync/completion.rs | 114 + rust/kernel/sync/condvar.rs | 258 + rust/kernel/sync/lock.rs | 324 + rust/kernel/sync/lock/global.rs | 312 + rust/kernel/sync/lock/mutex.rs | 147 + rust/kernel/sync/lock/spinlock.rs | 465 ++ rust/kernel/sync/locked_by.rs | 169 + rust/kernel/sync/poll.rs | 179 + rust/kernel/sync/rcu.rs | 88 + rust/kernel/sync/refcount.rs | 116 + rust/kernel/sync/set_once.rs | 133 + rust/kernel/sync/srcu.rs | 171 + rust/kernel/task.rs | 435 ++ rust/kernel/time.rs | 582 ++ rust/kernel/time/delay.rs | 86 + rust/kernel/time/hrtimer.rs | 1120 +++ rust/kernel/time/hrtimer/arc.rs | 111 + rust/kernel/time/hrtimer/pin.rs | 115 + rust/kernel/time/hrtimer/pin_mut.rs | 118 + rust/kernel/time/hrtimer/tbox.rs | 131 + rust/kernel/tracepoint.rs | 49 + rust/kernel/transmute.rs | 229 + rust/kernel/types.rs | 453 ++ rust/kernel/types/for_lt.rs | 169 + rust/kernel/uaccess.rs | 664 ++ rust/kernel/usb.rs | 485 ++ rust/kernel/workqueue.rs | 1114 +++ rust/kernel/xarray.rs | 283 + rust/macros/concat_idents.rs | 36 + rust/macros/export.rs | 25 + rust/macros/fmt.rs | 96 + rust/macros/for_lt.rs | 265 + rust/macros/helpers.rs | 60 + rust/macros/kunit.rs | 172 + rust/macros/lib.rs | 539 ++ rust/macros/module.rs | 654 ++ rust/macros/paste.rs | 113 + rust/macros/vtable.rs | 136 + rust/pin-init/CONTRIBUTING.md | 72 + rust/pin-init/README.md | 247 + rust/pin-init/examples/big_struct_in_place.rs | 43 + rust/pin-init/examples/error.rs | 31 + rust/pin-init/examples/linked_list.rs | 168 + rust/pin-init/examples/mutex.rs | 217 + rust/pin-init/examples/pthread_mutex.rs | 183 + rust/pin-init/examples/static_init.rs | 121 + rust/pin-init/internal/src/diagnostics.rs | 44 + rust/pin-init/internal/src/init.rs | 482 ++ rust/pin-init/internal/src/lib.rs | 59 + rust/pin-init/internal/src/pin_data.rs | 537 ++ rust/pin-init/internal/src/pinned_drop.rs | 61 + rust/pin-init/internal/src/zeroable.rs | 78 + rust/pin-init/src/__internal.rs | 409 ++ rust/pin-init/src/alloc.rs | 160 + rust/pin-init/src/lib.rs | 1797 +++++ rust/proc-macro2/README.md | 13 + rust/proc-macro2/detection.rs | 77 + rust/proc-macro2/extra.rs | 153 + rust/proc-macro2/fallback.rs | 1258 ++++ rust/proc-macro2/lib.rs | 1355 ++++ rust/proc-macro2/location.rs | 31 + rust/proc-macro2/marker.rs | 19 + rust/proc-macro2/parse.rs | 997 +++ rust/proc-macro2/probe.rs | 12 + rust/proc-macro2/probe/proc_macro_span.rs | 53 + rust/proc-macro2/probe/proc_macro_span_file.rs | 16 + rust/proc-macro2/probe/proc_macro_span_location.rs | 23 + rust/proc-macro2/rcvec.rs | 148 + rust/proc-macro2/wrapper.rs | 986 +++ rust/quote/README.md | 12 + rust/quote/ext.rs | 112 + rust/quote/format.rs | 170 + rust/quote/ident_fragment.rs | 90 + rust/quote/lib.rs | 1456 ++++ rust/quote/runtime.rs | 494 ++ rust/quote/spanned.rs | 52 + rust/quote/to_tokens.rs | 273 + rust/syn/README.md | 13 + rust/syn/attr.rs | 838 +++ rust/syn/bigint.rs | 68 + rust/syn/buffer.rs | 436 ++ rust/syn/classify.rs | 313 + rust/syn/custom_keyword.rs | 262 + rust/syn/custom_punctuation.rs | 306 + rust/syn/data.rs | 426 ++ rust/syn/derive.rs | 261 + rust/syn/discouraged.rs | 227 + rust/syn/drops.rs | 60 + rust/syn/error.rs | 469 ++ rust/syn/export.rs | 75 + rust/syn/expr.rs | 4175 +++++++++++ rust/syn/ext.rs | 138 + rust/syn/file.rs | 127 + rust/syn/fixup.rs | 775 ++ rust/syn/gen/clone.rs | 2269 ++++++ rust/syn/gen/debug.rs | 3240 +++++++++ rust/syn/gen/eq.rs | 2308 ++++++ rust/syn/gen/fold.rs | 3904 ++++++++++ rust/syn/gen/hash.rs | 2878 ++++++++ rust/syn/gen/visit.rs | 3943 ++++++++++ rust/syn/gen/visit_mut.rs | 3761 ++++++++++ rust/syn/generics.rs | 1479 ++++ rust/syn/group.rs | 293 + rust/syn/ident.rs | 110 + rust/syn/item.rs | 3492 +++++++++ rust/syn/lib.rs | 1013 +++ rust/syn/lifetime.rs | 158 + rust/syn/lit.rs | 1862 +++++ rust/syn/lookahead.rs | 334 + rust/syn/mac.rs | 227 + rust/syn/macros.rs | 184 + rust/syn/meta.rs | 429 ++ rust/syn/op.rs | 221 + rust/syn/parse.rs | 1421 ++++ rust/syn/parse_macro_input.rs | 130 + rust/syn/parse_quote.rs | 242 + rust/syn/pat.rs | 957 +++ rust/syn/path.rs | 968 +++ rust/syn/precedence.rs | 212 + rust/syn/print.rs | 18 + rust/syn/punctuated.rs | 1157 +++ rust/syn/restriction.rs | 180 + rust/syn/scan_expr.rs | 267 + rust/syn/sealed.rs | 6 + rust/syn/span.rs | 65 + rust/syn/spanned.rs | 120 + rust/syn/stmt.rs | 486 ++ rust/syn/thread.rs | 62 + rust/syn/token.rs | 1098 +++ rust/syn/tt.rs | 109 + rust/syn/ty.rs | 1273 ++++ rust/syn/verbatim.rs | 35 + rust/syn/whitespace.rs | 67 + rust/uapi/lib.rs | 42 + rust/uapi/uapi_helper.h | 17 + rust/zerocopy-derive/README.md | 14 + rust/zerocopy-derive/derive/from_bytes.rs | 192 + rust/zerocopy-derive/derive/into_bytes.rs | 164 + rust/zerocopy-derive/derive/known_layout.rs | 349 + rust/zerocopy-derive/derive/mod.rs | 136 + rust/zerocopy-derive/derive/try_from_bytes.rs | 765 ++ rust/zerocopy-derive/derive/unaligned.rs | 80 + rust/zerocopy-derive/lib.rs | 180 + rust/zerocopy-derive/repr.rs | 851 +++ rust/zerocopy-derive/util.rs | 863 +++ rust/zerocopy/README.md | 13 + rust/zerocopy/benches/as_bytes_dynamic_size.rs | 9 + rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 | 5 + .../benches/as_bytes_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/as_bytes_static_size.rs | 9 + rust/zerocopy/benches/as_bytes_static_size.x86-64 | 4 + .../benches/as_bytes_static_size.x86-64.mca | 45 + rust/zerocopy/benches/extend_vec_zeroed.rs | 9 + rust/zerocopy/benches/extend_vec_zeroed.x86-64 | 60 + rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca | 147 + .../benches/formats/coco_dynamic_padding.rs | 24 + rust/zerocopy/benches/formats/coco_dynamic_size.rs | 27 + rust/zerocopy/benches/formats/coco_static_size.rs | 27 + rust/zerocopy/benches/insert_vec_zeroed.rs | 13 + rust/zerocopy/benches/insert_vec_zeroed.x86-64 | 79 + rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca | 183 + rust/zerocopy/benches/new_box_zeroed.rs | 9 + rust/zerocopy/benches/new_box_zeroed.x86-64 | 7 + rust/zerocopy/benches/new_box_zeroed.x86-64.mca | 51 + .../new_box_zeroed_with_elems_dynamic_padding.rs | 11 + ...ew_box_zeroed_with_elems_dynamic_padding.x86-64 | 24 + ...ox_zeroed_with_elems_dynamic_padding.x86-64.mca | 81 + .../new_box_zeroed_with_elems_dynamic_size.rs | 9 + .../new_box_zeroed_with_elems_dynamic_size.x86-64 | 22 + ...w_box_zeroed_with_elems_dynamic_size.x86-64.mca | 77 + rust/zerocopy/benches/new_vec_zeroed.rs | 9 + rust/zerocopy/benches/new_vec_zeroed.x86-64 | 40 + rust/zerocopy/benches/new_vec_zeroed.x86-64.mca | 113 + rust/zerocopy/benches/new_zeroed.rs | 9 + rust/zerocopy/benches/new_zeroed.x86-64 | 3 + rust/zerocopy/benches/new_zeroed.x86-64.mca | 43 + rust/zerocopy/benches/read_from_bytes.rs | 7 + rust/zerocopy/benches/read_from_bytes.x86-64 | 15 + rust/zerocopy/benches/read_from_bytes.x86-64.mca | 65 + rust/zerocopy/benches/read_from_prefix.rs | 10 + rust/zerocopy/benches/read_from_prefix.x86-64 | 14 + rust/zerocopy/benches/read_from_prefix.x86-64.mca | 63 + rust/zerocopy/benches/read_from_suffix.rs | 10 + rust/zerocopy/benches/read_from_suffix.x86-64 | 15 + rust/zerocopy/benches/read_from_suffix.x86-64.mca | 65 + .../benches/ref_from_bytes_dynamic_padding.rs | 7 + .../benches/ref_from_bytes_dynamic_padding.x86-64 | 22 + .../ref_from_bytes_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_bytes_dynamic_size.rs | 7 + .../benches/ref_from_bytes_dynamic_size.x86-64 | 20 + .../benches/ref_from_bytes_dynamic_size.x86-64.mca | 75 + .../zerocopy/benches/ref_from_bytes_static_size.rs | 7 + .../benches/ref_from_bytes_static_size.x86-64 | 8 + .../benches/ref_from_bytes_static_size.x86-64.mca | 53 + .../ref_from_bytes_with_elems_dynamic_padding.rs | 10 + ...ef_from_bytes_with_elems_dynamic_padding.x86-64 | 19 + ...rom_bytes_with_elems_dynamic_padding.x86-64.mca | 71 + .../ref_from_bytes_with_elems_dynamic_size.rs | 10 + .../ref_from_bytes_with_elems_dynamic_size.x86-64 | 16 + ...f_from_bytes_with_elems_dynamic_size.x86-64.mca | 65 + .../benches/ref_from_prefix_dynamic_padding.rs | 10 + .../benches/ref_from_prefix_dynamic_padding.x86-64 | 22 + .../ref_from_prefix_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_prefix_dynamic_size.rs | 10 + .../benches/ref_from_prefix_dynamic_size.x86-64 | 17 + .../ref_from_prefix_dynamic_size.x86-64.mca | 67 + .../benches/ref_from_prefix_static_size.rs | 10 + .../benches/ref_from_prefix_static_size.x86-64 | 8 + .../benches/ref_from_prefix_static_size.x86-64.mca | 53 + .../ref_from_prefix_with_elems_dynamic_padding.rs | 13 + ...f_from_prefix_with_elems_dynamic_padding.x86-64 | 26 + ...om_prefix_with_elems_dynamic_padding.x86-64.mca | 85 + .../ref_from_prefix_with_elems_dynamic_size.rs | 13 + .../ref_from_prefix_with_elems_dynamic_size.x86-64 | 22 + ..._from_prefix_with_elems_dynamic_size.x86-64.mca | 77 + .../benches/ref_from_suffix_dynamic_padding.rs | 10 + .../benches/ref_from_suffix_dynamic_padding.x86-64 | 23 + .../ref_from_suffix_dynamic_padding.x86-64.mca | 79 + .../benches/ref_from_suffix_dynamic_size.rs | 10 + .../benches/ref_from_suffix_dynamic_size.x86-64 | 13 + .../ref_from_suffix_dynamic_size.x86-64.mca | 63 + .../benches/ref_from_suffix_static_size.rs | 10 + .../benches/ref_from_suffix_static_size.x86-64 | 13 + .../benches/ref_from_suffix_static_size.x86-64.mca | 61 + .../ref_from_suffix_with_elems_dynamic_padding.rs | 13 + ...f_from_suffix_with_elems_dynamic_padding.x86-64 | 27 + ...om_suffix_with_elems_dynamic_padding.x86-64.mca | 85 + .../ref_from_suffix_with_elems_dynamic_size.rs | 13 + .../ref_from_suffix_with_elems_dynamic_size.x86-64 | 23 + ..._from_suffix_with_elems_dynamic_size.x86-64.mca | 77 + rust/zerocopy/benches/split_at_dynamic_padding.rs | 12 + .../benches/split_at_dynamic_padding.x86-64 | 12 + .../benches/split_at_dynamic_padding.x86-64.mca | 59 + rust/zerocopy/benches/split_at_dynamic_size.rs | 12 + rust/zerocopy/benches/split_at_dynamic_size.x86-64 | 12 + .../benches/split_at_dynamic_size.x86-64.mca | 59 + .../benches/split_at_unchecked_dynamic_padding.rs | 12 + .../split_at_unchecked_dynamic_padding.x86-64 | 6 + .../split_at_unchecked_dynamic_padding.x86-64.mca | 49 + .../benches/split_at_unchecked_dynamic_size.rs | 12 + .../benches/split_at_unchecked_dynamic_size.x86-64 | 6 + .../split_at_unchecked_dynamic_size.x86-64.mca | 49 + .../benches/split_via_immutable_dynamic_padding.rs | 11 + .../split_via_immutable_dynamic_padding.x86-64 | 14 + .../split_via_immutable_dynamic_padding.x86-64.mca | 65 + .../benches/split_via_immutable_dynamic_size.rs | 11 + .../split_via_immutable_dynamic_size.x86-64 | 13 + .../split_via_immutable_dynamic_size.x86-64.mca | 63 + .../split_via_runtime_check_dynamic_padding.rs | 11 + .../split_via_runtime_check_dynamic_padding.x86-64 | 22 + ...it_via_runtime_check_dynamic_padding.x86-64.mca | 79 + .../split_via_runtime_check_dynamic_size.rs | 11 + .../split_via_runtime_check_dynamic_size.x86-64 | 13 + ...split_via_runtime_check_dynamic_size.x86-64.mca | 63 + .../benches/split_via_unchecked_dynamic_padding.rs | 11 + .../split_via_unchecked_dynamic_padding.x86-64 | 14 + .../split_via_unchecked_dynamic_padding.x86-64.mca | 65 + .../benches/split_via_unchecked_dynamic_size.rs | 11 + .../split_via_unchecked_dynamic_size.x86-64 | 13 + .../split_via_unchecked_dynamic_size.x86-64.mca | 63 + rust/zerocopy/benches/transmute.rs | 16 + rust/zerocopy/benches/transmute.x86-64 | 3 + rust/zerocopy/benches/transmute.x86-64.mca | 43 + .../zerocopy/benches/transmute_ref_dynamic_size.rs | 16 + .../benches/transmute_ref_dynamic_size.x86-64 | 4 + .../benches/transmute_ref_dynamic_size.x86-64.mca | 45 + rust/zerocopy/benches/transmute_ref_static_size.rs | 15 + .../benches/transmute_ref_static_size.x86-64 | 3 + .../benches/transmute_ref_static_size.x86-64.mca | 43 + rust/zerocopy/benches/try_read_from_bytes.rs | 7 + rust/zerocopy/benches/try_read_from_bytes.x86-64 | 23 + .../benches/try_read_from_bytes.x86-64.mca | 79 + rust/zerocopy/benches/try_read_from_prefix.rs | 10 + rust/zerocopy/benches/try_read_from_prefix.x86-64 | 16 + .../benches/try_read_from_prefix.x86-64.mca | 67 + rust/zerocopy/benches/try_read_from_suffix.rs | 10 + rust/zerocopy/benches/try_read_from_suffix.x86-64 | 18 + .../benches/try_read_from_suffix.x86-64.mca | 71 + .../benches/try_ref_from_bytes_dynamic_padding.rs | 7 + .../try_ref_from_bytes_dynamic_padding.x86-64 | 24 + .../try_ref_from_bytes_dynamic_padding.x86-64.mca | 81 + .../benches/try_ref_from_bytes_dynamic_size.rs | 7 + .../benches/try_ref_from_bytes_dynamic_size.x86-64 | 22 + .../try_ref_from_bytes_dynamic_size.x86-64.mca | 79 + .../benches/try_ref_from_bytes_static_size.rs | 7 + .../benches/try_ref_from_bytes_static_size.x86-64 | 13 + .../try_ref_from_bytes_static_size.x86-64.mca | 59 + ...ry_ref_from_bytes_with_elems_dynamic_padding.rs | 10 + ...ef_from_bytes_with_elems_dynamic_padding.x86-64 | 21 + ...rom_bytes_with_elems_dynamic_padding.x86-64.mca | 75 + .../try_ref_from_bytes_with_elems_dynamic_size.rs | 10 + ...y_ref_from_bytes_with_elems_dynamic_size.x86-64 | 18 + ...f_from_bytes_with_elems_dynamic_size.x86-64.mca | 69 + .../benches/try_ref_from_prefix_dynamic_padding.rs | 10 + .../try_ref_from_prefix_dynamic_padding.x86-64 | 29 + .../try_ref_from_prefix_dynamic_padding.x86-64.mca | 91 + .../benches/try_ref_from_prefix_dynamic_size.rs | 10 + .../try_ref_from_prefix_dynamic_size.x86-64 | 22 + .../try_ref_from_prefix_dynamic_size.x86-64.mca | 77 + .../benches/try_ref_from_prefix_static_size.rs | 10 + .../benches/try_ref_from_prefix_static_size.x86-64 | 15 + .../try_ref_from_prefix_static_size.x86-64.mca | 63 + ...y_ref_from_prefix_with_elems_dynamic_padding.rs | 13 + ...f_from_prefix_with_elems_dynamic_padding.x86-64 | 30 + ...om_prefix_with_elems_dynamic_padding.x86-64.mca | 91 + .../try_ref_from_prefix_with_elems_dynamic_size.rs | 13 + ..._ref_from_prefix_with_elems_dynamic_size.x86-64 | 26 + ..._from_prefix_with_elems_dynamic_size.x86-64.mca | 83 + .../benches/try_ref_from_suffix_dynamic_padding.rs | 10 + .../try_ref_from_suffix_dynamic_padding.x86-64 | 26 + .../try_ref_from_suffix_dynamic_padding.x86-64.mca | 85 + .../benches/try_ref_from_suffix_dynamic_size.rs | 10 + .../try_ref_from_suffix_dynamic_size.x86-64 | 18 + .../try_ref_from_suffix_dynamic_size.x86-64.mca | 71 + .../benches/try_ref_from_suffix_static_size.rs | 10 + .../benches/try_ref_from_suffix_static_size.x86-64 | 16 + .../try_ref_from_suffix_static_size.x86-64.mca | 67 + ...y_ref_from_suffix_with_elems_dynamic_padding.rs | 13 + ...f_from_suffix_with_elems_dynamic_padding.x86-64 | 32 + ...om_suffix_with_elems_dynamic_padding.x86-64.mca | 95 + .../try_ref_from_suffix_with_elems_dynamic_size.rs | 13 + ..._ref_from_suffix_with_elems_dynamic_size.x86-64 | 28 + ..._from_suffix_with_elems_dynamic_size.x86-64.mca | 87 + rust/zerocopy/benches/try_transmute.rs | 16 + rust/zerocopy/benches/try_transmute.x86-64 | 9 + rust/zerocopy/benches/try_transmute.x86-64.mca | 55 + .../benches/try_transmute_ref_dynamic_size.rs | 18 + .../benches/try_transmute_ref_dynamic_size.x86-64 | 6 + .../try_transmute_ref_dynamic_size.x86-64.mca | 49 + .../benches/try_transmute_ref_static_size.rs | 17 + .../benches/try_transmute_ref_static_size.x86-64 | 5 + .../try_transmute_ref_static_size.x86-64.mca | 47 + rust/zerocopy/benches/write_to_dynamic_size.rs | 9 + rust/zerocopy/benches/write_to_dynamic_size.x86-64 | 21 + .../benches/write_to_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_dynamic_size.rs | 12 + .../benches/write_to_prefix_dynamic_size.x86-64 | 21 + .../write_to_prefix_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_static_size.rs | 12 + .../benches/write_to_prefix_static_size.x86-64 | 11 + .../benches/write_to_prefix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/write_to_static_size.rs | 9 + rust/zerocopy/benches/write_to_static_size.x86-64 | 11 + .../benches/write_to_static_size.x86-64.mca | 57 + .../benches/write_to_suffix_dynamic_size.rs | 12 + .../benches/write_to_suffix_dynamic_size.x86-64 | 22 + .../write_to_suffix_dynamic_size.x86-64.mca | 79 + .../benches/write_to_suffix_static_size.rs | 12 + .../benches/write_to_suffix_static_size.x86-64 | 11 + .../benches/write_to_suffix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/zero_dynamic_padding.rs | 9 + rust/zerocopy/benches/zero_dynamic_padding.x86-64 | 7 + .../benches/zero_dynamic_padding.x86-64.mca | 51 + rust/zerocopy/benches/zero_dynamic_size.rs | 9 + rust/zerocopy/benches/zero_dynamic_size.x86-64 | 5 + rust/zerocopy/benches/zero_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/zero_static_size.rs | 9 + rust/zerocopy/benches/zero_static_size.x86-64 | 4 + rust/zerocopy/benches/zero_static_size.x86-64.mca | 45 + rust/zerocopy/rustdoc/style.css | 56 + rust/zerocopy/src/byte_slice.rs | 434 ++ rust/zerocopy/src/byteorder.rs | 1596 ++++ rust/zerocopy/src/deprecated.rs | 281 + rust/zerocopy/src/error.rs | 1350 ++++ rust/zerocopy/src/impls.rs | 2389 ++++++ rust/zerocopy/src/layout.rs | 2227 ++++++ rust/zerocopy/src/lib.rs | 7614 ++++++++++++++++++++ rust/zerocopy/src/macros.rs | 1825 +++++ rust/zerocopy/src/pointer/inner.rs | 754 ++ rust/zerocopy/src/pointer/invariant.rs | 298 + rust/zerocopy/src/pointer/mod.rs | 410 ++ rust/zerocopy/src/pointer/ptr.rs | 1586 ++++ rust/zerocopy/src/pointer/transmute.rs | 522 ++ rust/zerocopy/src/ref.rs | 1358 ++++ rust/zerocopy/src/split_at.rs | 1090 +++ rust/zerocopy/src/util/macro_util.rs | 1310 ++++ rust/zerocopy/src/util/macros.rs | 1067 +++ rust/zerocopy/src/util/mod.rs | 944 +++ rust/zerocopy/src/wrappers.rs | 1034 +++ 596 files changed, 168177 insertions(+) create mode 100644 rust/.gitignore create mode 100644 rust/.kunitconfig create mode 100644 rust/Makefile create mode 100644 rust/bindgen_parameters create mode 100644 rust/bindings/bindings_helper.h create mode 100644 rust/bindings/lib.rs create mode 100644 rust/build_error.rs create mode 100644 rust/compiler_builtins.rs create mode 100644 rust/exports.c create mode 100644 rust/ffi.rs create mode 100644 rust/helpers/atomic.c create mode 100644 rust/helpers/atomic_ext.c create mode 100644 rust/helpers/auxiliary.c create mode 100644 rust/helpers/barrier.c create mode 100644 rust/helpers/binder.c create mode 100644 rust/helpers/bitmap.c create mode 100644 rust/helpers/bitops.c create mode 100644 rust/helpers/blk.c create mode 100644 rust/helpers/bug.c create mode 100644 rust/helpers/build_assert.c create mode 100644 rust/helpers/build_bug.c create mode 100644 rust/helpers/clk.c create mode 100644 rust/helpers/completion.c create mode 100644 rust/helpers/cpu.c create mode 100644 rust/helpers/cpufreq.c create mode 100644 rust/helpers/cpumask.c create mode 100644 rust/helpers/cred.c create mode 100644 rust/helpers/device.c create mode 100644 rust/helpers/dma-resv.c create mode 100644 rust/helpers/dma.c create mode 100644 rust/helpers/drm.c create mode 100644 rust/helpers/drm_gpuvm.c create mode 100644 rust/helpers/err.c create mode 100644 rust/helpers/fs.c create mode 100644 rust/helpers/fwctl.c create mode 100644 rust/helpers/gpu.c create mode 100644 rust/helpers/helpers.c create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/helpers/io.c create mode 100644 rust/helpers/irq.c create mode 100644 rust/helpers/jump_label.c create mode 100644 rust/helpers/kunit.c create mode 100644 rust/helpers/list.c create mode 100644 rust/helpers/maple_tree.c create mode 100644 rust/helpers/mm.c create mode 100644 rust/helpers/mutex.c create mode 100644 rust/helpers/net/genetlink.c create mode 100644 rust/helpers/of.c create mode 100644 rust/helpers/page.c create mode 100644 rust/helpers/pci.c create mode 100644 rust/helpers/pid_namespace.c create mode 100644 rust/helpers/platform.c create mode 100644 rust/helpers/poll.c create mode 100644 rust/helpers/processor.c create mode 100644 rust/helpers/property.c create mode 100644 rust/helpers/pwm.c create mode 100644 rust/helpers/rbtree.c create mode 100644 rust/helpers/rcu.c create mode 100644 rust/helpers/refcount.c create mode 100644 rust/helpers/regulator.c create mode 100644 rust/helpers/scatterlist.c create mode 100644 rust/helpers/security.c create mode 100644 rust/helpers/serdev.c create mode 100644 rust/helpers/signal.c create mode 100644 rust/helpers/slab.c create mode 100644 rust/helpers/spinlock.c create mode 100644 rust/helpers/srcu.c create mode 100644 rust/helpers/string.c create mode 100644 rust/helpers/sync.c create mode 100644 rust/helpers/task.c create mode 100644 rust/helpers/time.c create mode 100644 rust/helpers/uaccess.c create mode 100644 rust/helpers/usb.c create mode 100644 rust/helpers/vmalloc.c create mode 100644 rust/helpers/wait.c create mode 100644 rust/helpers/workqueue.c create mode 100644 rust/helpers/xarray.c create mode 100644 rust/kernel/.gitignore create mode 100644 rust/kernel/Kconfig.test create mode 100644 rust/kernel/acpi.rs create mode 100644 rust/kernel/alloc.rs create mode 100644 rust/kernel/alloc/allocator.rs create mode 100644 rust/kernel/alloc/allocator/iter.rs create mode 100644 rust/kernel/alloc/kbox.rs create mode 100644 rust/kernel/alloc/kvec.rs create mode 100644 rust/kernel/alloc/kvec/errors.rs create mode 100644 rust/kernel/alloc/layout.rs create mode 100644 rust/kernel/auxiliary.rs create mode 100644 rust/kernel/bitfield.rs create mode 100644 rust/kernel/bitmap.rs create mode 100644 rust/kernel/bits.rs create mode 100644 rust/kernel/block.rs create mode 100644 rust/kernel/block/mq.rs create mode 100644 rust/kernel/block/mq/gen_disk.rs create mode 100644 rust/kernel/block/mq/operations.rs create mode 100644 rust/kernel/block/mq/request.rs create mode 100644 rust/kernel/block/mq/tag_set.rs create mode 100644 rust/kernel/bug.rs create mode 100644 rust/kernel/build_assert.rs create mode 100644 rust/kernel/clk.rs create mode 100644 rust/kernel/configfs.rs create mode 100644 rust/kernel/cpu.rs create mode 100644 rust/kernel/cpufreq.rs create mode 100644 rust/kernel/cpumask.rs create mode 100644 rust/kernel/cred.rs create mode 100644 rust/kernel/debugfs.rs create mode 100644 rust/kernel/debugfs/callback_adapters.rs create mode 100644 rust/kernel/debugfs/entry.rs create mode 100644 rust/kernel/debugfs/file_ops.rs create mode 100644 rust/kernel/debugfs/traits.rs create mode 100644 rust/kernel/device.rs create mode 100644 rust/kernel/device/property.rs create mode 100644 rust/kernel/device_id.rs create mode 100644 rust/kernel/devres.rs create mode 100644 rust/kernel/dma.rs create mode 100644 rust/kernel/driver.rs create mode 100644 rust/kernel/drm/device.rs create mode 100644 rust/kernel/drm/driver.rs create mode 100644 rust/kernel/drm/file.rs create mode 100644 rust/kernel/drm/gem/mod.rs create mode 100644 rust/kernel/drm/gem/shmem.rs create mode 100644 rust/kernel/drm/gpuvm/mod.rs create mode 100644 rust/kernel/drm/gpuvm/sm_ops.rs create mode 100644 rust/kernel/drm/gpuvm/va.rs create mode 100644 rust/kernel/drm/gpuvm/vm_bo.rs create mode 100644 rust/kernel/drm/ioctl.rs create mode 100644 rust/kernel/drm/mod.rs create mode 100644 rust/kernel/error.rs create mode 100644 rust/kernel/faux.rs create mode 100644 rust/kernel/firmware.rs create mode 100644 rust/kernel/fmt.rs create mode 100644 rust/kernel/fs.rs create mode 100644 rust/kernel/fs/file.rs create mode 100644 rust/kernel/fs/kiocb.rs create mode 100644 rust/kernel/fwctl.rs create mode 100644 rust/kernel/generated_arch_reachable_asm.rs.S create mode 100644 rust/kernel/generated_arch_static_branch_asm.rs.S create mode 100644 rust/kernel/generated_arch_warn_asm.rs.S create mode 100644 rust/kernel/gpu.rs create mode 100644 rust/kernel/gpu/buddy.rs create mode 100644 rust/kernel/i2c.rs create mode 100644 rust/kernel/id_pool.rs create mode 100644 rust/kernel/impl_flags.rs create mode 100644 rust/kernel/init.rs create mode 100644 rust/kernel/interop.rs create mode 100644 rust/kernel/interop/list.rs create mode 100644 rust/kernel/interrupt.rs create mode 100644 rust/kernel/io.rs create mode 100644 rust/kernel/io/mem.rs create mode 100644 rust/kernel/io/poll.rs create mode 100644 rust/kernel/io/register.rs create mode 100644 rust/kernel/io/resource.rs create mode 100644 rust/kernel/ioctl.rs create mode 100644 rust/kernel/iommu/mod.rs create mode 100644 rust/kernel/iommu/pgtable.rs create mode 100644 rust/kernel/iov.rs create mode 100644 rust/kernel/irq.rs create mode 100644 rust/kernel/irq/flags.rs create mode 100644 rust/kernel/irq/request.rs create mode 100644 rust/kernel/jump_label.rs create mode 100644 rust/kernel/kunit.rs create mode 100644 rust/kernel/lib.rs create mode 100644 rust/kernel/list.rs create mode 100644 rust/kernel/list/arc.rs create mode 100644 rust/kernel/list/arc_field.rs create mode 100644 rust/kernel/list/impl_list_item_mod.rs create mode 100644 rust/kernel/maple_tree.rs create mode 100644 rust/kernel/miscdevice.rs create mode 100644 rust/kernel/mm.rs create mode 100644 rust/kernel/mm/mmput_async.rs create mode 100644 rust/kernel/mm/virt.rs create mode 100644 rust/kernel/module.rs create mode 100644 rust/kernel/module_param.rs create mode 100644 rust/kernel/net/mod.rs create mode 100644 rust/kernel/net/netlink.rs create mode 100644 rust/kernel/net/phy.rs create mode 100644 rust/kernel/net/phy/reg.rs create mode 100644 rust/kernel/num.rs create mode 100644 rust/kernel/num/bounded.rs create mode 100644 rust/kernel/num/casts.rs create mode 100644 rust/kernel/of.rs create mode 100644 rust/kernel/opp.rs create mode 100644 rust/kernel/page.rs create mode 100644 rust/kernel/pci.rs create mode 100644 rust/kernel/pci/id.rs create mode 100644 rust/kernel/pci/io.rs create mode 100644 rust/kernel/pci/irq.rs create mode 100644 rust/kernel/pid_namespace.rs create mode 100644 rust/kernel/platform.rs create mode 100644 rust/kernel/prelude.rs create mode 100644 rust/kernel/print.rs create mode 100644 rust/kernel/processor.rs create mode 100644 rust/kernel/ptr.rs create mode 100644 rust/kernel/ptr/projection.rs create mode 100644 rust/kernel/pwm.rs create mode 100644 rust/kernel/rbtree.rs create mode 100644 rust/kernel/regulator.rs create mode 100644 rust/kernel/revocable.rs create mode 100644 rust/kernel/safety.rs create mode 100644 rust/kernel/scatterlist.rs create mode 100644 rust/kernel/security.rs create mode 100644 rust/kernel/seq_file.rs create mode 100644 rust/kernel/serdev.rs create mode 100644 rust/kernel/sizes.rs create mode 100644 rust/kernel/soc.rs create mode 100644 rust/kernel/std_vendor.rs create mode 100644 rust/kernel/str.rs create mode 100644 rust/kernel/str/parse_int.rs create mode 100644 rust/kernel/sync.rs create mode 100644 rust/kernel/sync/arc.rs create mode 100644 rust/kernel/sync/arc/std_vendor.rs create mode 100644 rust/kernel/sync/aref.rs create mode 100644 rust/kernel/sync/atomic.rs create mode 100644 rust/kernel/sync/atomic/internal.rs create mode 100644 rust/kernel/sync/atomic/ordering.rs create mode 100644 rust/kernel/sync/atomic/predefine.rs create mode 100644 rust/kernel/sync/barrier.rs create mode 100644 rust/kernel/sync/completion.rs create mode 100644 rust/kernel/sync/condvar.rs create mode 100644 rust/kernel/sync/lock.rs create mode 100644 rust/kernel/sync/lock/global.rs create mode 100644 rust/kernel/sync/lock/mutex.rs create mode 100644 rust/kernel/sync/lock/spinlock.rs create mode 100644 rust/kernel/sync/locked_by.rs create mode 100644 rust/kernel/sync/poll.rs create mode 100644 rust/kernel/sync/rcu.rs create mode 100644 rust/kernel/sync/refcount.rs create mode 100644 rust/kernel/sync/set_once.rs create mode 100644 rust/kernel/sync/srcu.rs create mode 100644 rust/kernel/task.rs create mode 100644 rust/kernel/time.rs create mode 100644 rust/kernel/time/delay.rs create mode 100644 rust/kernel/time/hrtimer.rs create mode 100644 rust/kernel/time/hrtimer/arc.rs create mode 100644 rust/kernel/time/hrtimer/pin.rs create mode 100644 rust/kernel/time/hrtimer/pin_mut.rs create mode 100644 rust/kernel/time/hrtimer/tbox.rs create mode 100644 rust/kernel/tracepoint.rs create mode 100644 rust/kernel/transmute.rs create mode 100644 rust/kernel/types.rs create mode 100644 rust/kernel/types/for_lt.rs create mode 100644 rust/kernel/uaccess.rs create mode 100644 rust/kernel/usb.rs create mode 100644 rust/kernel/workqueue.rs create mode 100644 rust/kernel/xarray.rs create mode 100644 rust/macros/concat_idents.rs create mode 100644 rust/macros/export.rs create mode 100644 rust/macros/fmt.rs create mode 100644 rust/macros/for_lt.rs create mode 100644 rust/macros/helpers.rs create mode 100644 rust/macros/kunit.rs create mode 100644 rust/macros/lib.rs create mode 100644 rust/macros/module.rs create mode 100644 rust/macros/paste.rs create mode 100644 rust/macros/vtable.rs create mode 100644 rust/pin-init/CONTRIBUTING.md create mode 100644 rust/pin-init/README.md create mode 100644 rust/pin-init/examples/big_struct_in_place.rs create mode 100644 rust/pin-init/examples/error.rs create mode 100644 rust/pin-init/examples/linked_list.rs create mode 100644 rust/pin-init/examples/mutex.rs create mode 100644 rust/pin-init/examples/pthread_mutex.rs create mode 100644 rust/pin-init/examples/static_init.rs create mode 100644 rust/pin-init/internal/src/diagnostics.rs create mode 100644 rust/pin-init/internal/src/init.rs create mode 100644 rust/pin-init/internal/src/lib.rs create mode 100644 rust/pin-init/internal/src/pin_data.rs create mode 100644 rust/pin-init/internal/src/pinned_drop.rs create mode 100644 rust/pin-init/internal/src/zeroable.rs create mode 100644 rust/pin-init/src/__internal.rs create mode 100644 rust/pin-init/src/alloc.rs create mode 100644 rust/pin-init/src/lib.rs create mode 100644 rust/proc-macro2/README.md create mode 100644 rust/proc-macro2/detection.rs create mode 100644 rust/proc-macro2/extra.rs create mode 100644 rust/proc-macro2/fallback.rs create mode 100644 rust/proc-macro2/lib.rs create mode 100644 rust/proc-macro2/location.rs create mode 100644 rust/proc-macro2/marker.rs create mode 100644 rust/proc-macro2/parse.rs create mode 100644 rust/proc-macro2/probe.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_file.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_location.rs create mode 100644 rust/proc-macro2/rcvec.rs create mode 100644 rust/proc-macro2/wrapper.rs create mode 100644 rust/quote/README.md create mode 100644 rust/quote/ext.rs create mode 100644 rust/quote/format.rs create mode 100644 rust/quote/ident_fragment.rs create mode 100644 rust/quote/lib.rs create mode 100644 rust/quote/runtime.rs create mode 100644 rust/quote/spanned.rs create mode 100644 rust/quote/to_tokens.rs create mode 100644 rust/syn/README.md create mode 100644 rust/syn/attr.rs create mode 100644 rust/syn/bigint.rs create mode 100644 rust/syn/buffer.rs create mode 100644 rust/syn/classify.rs create mode 100644 rust/syn/custom_keyword.rs create mode 100644 rust/syn/custom_punctuation.rs create mode 100644 rust/syn/data.rs create mode 100644 rust/syn/derive.rs create mode 100644 rust/syn/discouraged.rs create mode 100644 rust/syn/drops.rs create mode 100644 rust/syn/error.rs create mode 100644 rust/syn/export.rs create mode 100644 rust/syn/expr.rs create mode 100644 rust/syn/ext.rs create mode 100644 rust/syn/file.rs create mode 100644 rust/syn/fixup.rs create mode 100644 rust/syn/gen/clone.rs create mode 100644 rust/syn/gen/debug.rs create mode 100644 rust/syn/gen/eq.rs create mode 100644 rust/syn/gen/fold.rs create mode 100644 rust/syn/gen/hash.rs create mode 100644 rust/syn/gen/visit.rs create mode 100644 rust/syn/gen/visit_mut.rs create mode 100644 rust/syn/generics.rs create mode 100644 rust/syn/group.rs create mode 100644 rust/syn/ident.rs create mode 100644 rust/syn/item.rs create mode 100644 rust/syn/lib.rs create mode 100644 rust/syn/lifetime.rs create mode 100644 rust/syn/lit.rs create mode 100644 rust/syn/lookahead.rs create mode 100644 rust/syn/mac.rs create mode 100644 rust/syn/macros.rs create mode 100644 rust/syn/meta.rs create mode 100644 rust/syn/op.rs create mode 100644 rust/syn/parse.rs create mode 100644 rust/syn/parse_macro_input.rs create mode 100644 rust/syn/parse_quote.rs create mode 100644 rust/syn/pat.rs create mode 100644 rust/syn/path.rs create mode 100644 rust/syn/precedence.rs create mode 100644 rust/syn/print.rs create mode 100644 rust/syn/punctuated.rs create mode 100644 rust/syn/restriction.rs create mode 100644 rust/syn/scan_expr.rs create mode 100644 rust/syn/sealed.rs create mode 100644 rust/syn/span.rs create mode 100644 rust/syn/spanned.rs create mode 100644 rust/syn/stmt.rs create mode 100644 rust/syn/thread.rs create mode 100644 rust/syn/token.rs create mode 100644 rust/syn/tt.rs create mode 100644 rust/syn/ty.rs create mode 100644 rust/syn/verbatim.rs create mode 100644 rust/syn/whitespace.rs create mode 100644 rust/uapi/lib.rs create mode 100644 rust/uapi/uapi_helper.h create mode 100644 rust/zerocopy-derive/README.md create mode 100644 rust/zerocopy-derive/derive/from_bytes.rs create mode 100644 rust/zerocopy-derive/derive/into_bytes.rs create mode 100644 rust/zerocopy-derive/derive/known_layout.rs create mode 100644 rust/zerocopy-derive/derive/mod.rs create mode 100644 rust/zerocopy-derive/derive/try_from_bytes.rs create mode 100644 rust/zerocopy-derive/derive/unaligned.rs create mode 100644 rust/zerocopy-derive/lib.rs create mode 100644 rust/zerocopy-derive/repr.rs create mode 100644 rust/zerocopy-derive/util.rs create mode 100644 rust/zerocopy/README.md create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/as_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_size.rs create mode 100644 rust/zerocopy/benches/formats/coco_static_size.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/new_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_zeroed.rs create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_bytes.rs create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_prefix.rs create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_suffix.rs create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute.rs create mode 100644 rust/zerocopy/benches/transmute.x86-64 create mode 100644 rust/zerocopy/benches/transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_bytes.rs create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_prefix.rs create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_suffix.rs create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute.rs create mode 100644 rust/zerocopy/benches/try_transmute.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_size.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_static_size.rs create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64.mca create mode 100644 rust/zerocopy/rustdoc/style.css create mode 100644 rust/zerocopy/src/byte_slice.rs create mode 100644 rust/zerocopy/src/byteorder.rs create mode 100644 rust/zerocopy/src/deprecated.rs create mode 100644 rust/zerocopy/src/error.rs create mode 100644 rust/zerocopy/src/impls.rs create mode 100644 rust/zerocopy/src/layout.rs create mode 100644 rust/zerocopy/src/lib.rs create mode 100644 rust/zerocopy/src/macros.rs create mode 100644 rust/zerocopy/src/pointer/inner.rs create mode 100644 rust/zerocopy/src/pointer/invariant.rs create mode 100644 rust/zerocopy/src/pointer/mod.rs create mode 100644 rust/zerocopy/src/pointer/ptr.rs create mode 100644 rust/zerocopy/src/pointer/transmute.rs create mode 100644 rust/zerocopy/src/ref.rs create mode 100644 rust/zerocopy/src/split_at.rs create mode 100644 rust/zerocopy/src/util/macro_util.rs create mode 100644 rust/zerocopy/src/util/macros.rs create mode 100644 rust/zerocopy/src/util/mod.rs create mode 100644 rust/zerocopy/src/wrappers.rs (limited to 'rust') diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 000000000..d3829ffab --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 + +bindings_generated.rs +bindings_helpers_generated.rs +doctests_kernel_generated.rs +doctests_kernel_generated_kunit.c +uapi_generated.rs +exports_*_generated.h +doc/ +test/ diff --git a/rust/.kunitconfig b/rust/.kunitconfig new file mode 100644 index 000000000..9e72a5ab0 --- /dev/null +++ b/rust/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_RUST=y +CONFIG_RUST_KERNEL_DOCTESTS=y diff --git a/rust/Makefile b/rust/Makefile new file mode 100644 index 000000000..da1a7409d --- /dev/null +++ b/rust/Makefile @@ -0,0 +1,815 @@ +# SPDX-License-Identifier: GPL-2.0 + +# Where to place rustdoc generated documentation +rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc + +# Clean generated host directory +clean-files := host/ + +obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o +always-$(CONFIG_RUST) += exports_core_generated.h + +obj-$(CONFIG_RUST) += zerocopy.o + +ifdef CONFIG_RUST_INLINE_HELPERS +always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc +else +obj-$(CONFIG_RUST) += helpers/helpers.o +always-$(CONFIG_RUST) += exports_helpers_generated.h +endif +# Missing prototypes are expected in the helpers since these are exported +# for Rust only, thus there is no header nor prototypes. +CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations + +always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs +obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o +always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h + +always-$(CONFIG_RUST) += uapi/uapi_generated.rs +obj-$(CONFIG_RUST) += uapi.o + +ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW +obj-$(CONFIG_RUST) += build_error.o +else +always-$(CONFIG_RUST) += build_error.o +endif + +obj-$(CONFIG_RUST) += exports.o + +always-$(CONFIG_RUST) += host/libproc_macro2.rlib host/libquote.rlib host/libsyn.rlib + +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c + +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o + +always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs +ifndef CONFIG_UML +always-$(subst y,$(CONFIG_RUST),$(CONFIG_BUG)) += kernel/generated_arch_warn_asm.rs kernel/generated_arch_reachable_asm.rs +endif + +# Avoids running `$(RUSTC)` when it may not be available. +ifdef CONFIG_RUST + +procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - /dev/null +endif + +cfgs-to-flags = $(patsubst %,--cfg='%',$1) + +core-cfgs := \ + no_fp_fmt_parse + +core-edition := $(if $(call rustc-min-version,108700),2024,2021) + +core-skip_flags := \ + --edition=2021 \ + -Wunreachable_pub + +core-flags := \ + --edition=$(core-edition) \ + $(call cfgs-to-flags,$(core-cfgs)) + +zerocopy-cfgs := \ + no_fp_fmt_parse + +zerocopy-flags := \ + --cap-lints=allow \ + $(call cfgs-to-flags,$(zerocopy-cfgs)) + +zerocopy-envs := \ + CARGO_PKG_VERSION=0.8.54 + +proc_macro2-cfgs := \ + feature="proc-macro" \ + wrap_proc_macro \ + $(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location) + +proc_macro2-flags := \ + --cap-lints=allow \ + $(call cfgs-to-flags,$(proc_macro2-cfgs)) + +quote-cfgs := \ + feature="proc-macro" + +quote-skip_flags := \ + --edition=2021 + +quote-flags := \ + --edition=2018 \ + --cap-lints=allow \ + --extern proc_macro2 \ + $(call cfgs-to-flags,$(quote-cfgs)) + +# `extra-traits`, `fold` and `visit` may be enabled if needed. +syn-cfgs := \ + feature="clone-impls" \ + feature="derive" \ + feature="full" \ + feature="parsing" \ + feature="printing" \ + feature="proc-macro" \ + feature="visit" \ + feature="visit-mut" + +syn-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + $(call cfgs-to-flags,$(syn-cfgs)) + +zerocopy_derive-cfgs := \ + zerocopy_unstable_linux + +zerocopy_derive-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + --extern syn \ + $(call cfgs-to-flags,$(zerocopy_derive-cfgs)) + +pin_init_internal-cfgs := \ + kernel USE_RUSTC_FEATURES + +pin_init_internal-flags := \ + --extern proc_macro2 \ + --extern quote \ + --extern syn \ + $(call cfgs-to-flags,$(pin_init_internal-cfgs)) + +pin_init-cfgs := \ + kernel USE_RUSTC_FEATURES + +pin_init-flags := \ + --extern pin_init_internal \ + --extern macros \ + $(call cfgs-to-flags,$(pin_init-cfgs)) + +# `rustdoc` did not save the target modifiers, thus workaround for +# the time being (https://github.com/rust-lang/rust/issues/144521). +rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18) + +# Similarly, for doctests (https://github.com/rust-lang/rust/issues/146465). +doctests_modifiers_workaround := $(rustdoc_modifiers_workaround)$(if $(call rustc-min-version,109100),$(comma)sanitizer) + +quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< + cmd_rustdoc = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ + $(rustc_target_flags) -L$(objtree)/$(obj)$(if $(rustdoc_host),/host) \ + -Zunstable-options --generate-link-to-definition \ + --output $(rustdoc_output) \ + --crate-name $(subst rustdoc-,,$@) \ + $(rustdoc_modifiers_workaround) \ + $(if $(rustdoc_host),,--sysroot=/dev/null) \ + @$(objtree)/include/generated/rustc_cfg $< + +# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute +# can be used to specify a custom logo. However: +# - The given value is used as-is, thus it cannot be relative or a local file +# (unlike the non-custom case) since the generated docs have subfolders. +# - It requires adding it to every crate. +# - It requires changing `core` which comes from the sysroot. +# +# Using `-Zcrate-attr` would solve the last two points, but not the first. +# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new +# command-like flags to solve the issue. Meanwhile, we use the non-custom case +# and then retouch the generated files. +rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ + rustdoc-kernel rustdoc-pin_init rustdoc-zerocopy rustdoc-zerocopy_derive + $(Q)grep -Ehro '::g' \ + -e 's:::g' + $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \ + echo ".logo-container > img { object-fit: contain; }" >> $$f; done + +rustdoc-proc_macro2: private rustdoc_host = yes +rustdoc-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rustdoc-proc_macro2: $(src)/proc-macro2/lib.rs rustdoc-clean FORCE + +$(call if_changed,rustdoc) + +rustdoc-quote: private rustdoc_host = yes +rustdoc-quote: private rustc_target_flags = $(quote-flags) +rustdoc-quote: private skip_flags = $(quote-skip_flags) +rustdoc-quote: $(src)/quote/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE + +$(call if_changed,rustdoc) + +rustdoc-syn: private rustdoc_host = yes +rustdoc-syn: private rustc_target_flags = $(syn-flags) +rustdoc-syn: $(src)/syn/lib.rs rustdoc-clean rustdoc-quote FORCE + +$(call if_changed,rustdoc) + +rustdoc-zerocopy_derive: private rustdoc_host = yes +rustdoc-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rustdoc-clean rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-macros: private rustdoc_host = yes +rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ + --extern proc_macro --extern proc_macro2 --extern quote --extern syn +rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 \ + rustdoc-quote rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-core: private skip_flags = $(core-skip_flags) +rustdoc-core: private rustc_target_flags = $(core-flags) +rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs rustdoc-clean FORCE + +$(call if_changed,rustdoc) + +# Even if `rustdoc` targets are not kernel objects, they should still be +# treated as such so that we pass the same flags. Otherwise, for instance, +# `rustdoc` will complain about missing sanitizer flags causing an ABI mismatch. +rustdoc-compiler_builtins: private is-kernel-object := y +rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rustdoc-zerocopy: private is-kernel-object := y +rustdoc-zerocopy: private rustc_target_flags = $(zerocopy-flags) \ + --extend-css $(src)/zerocopy/rustdoc/style.css +rustdoc-zerocopy: $(src)/zerocopy/src/lib.rs rustdoc-clean rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-ffi: private is-kernel-object := y +rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE + +$(call if_changed,rustdoc) + +rustdoc-pin_init_internal: private rustdoc_host = yes +rustdoc-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ + rustdoc-clean rustdoc-proc_macro2 rustdoc-quote rustdoc-syn FORCE + +$(call if_changed,rustdoc) + +rustdoc-pin_init: private rustdoc_host = yes +rustdoc-pin_init: private rustc_target_flags = $(pin_init-flags) \ + --extern pin_init_internal=$(objtree)/$(obj)/$(libpin_init_internal_name) \ + --extern alloc --cfg feature=\"alloc\" +rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ + rustdoc-macros FORCE + +$(call if_changed,rustdoc) + +rustdoc-kernel: private is-kernel-object := y +rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ + --extern build_error --extern macros \ + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive +rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ + rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \ + $(obj)/bindings.o FORCE + +$(call if_changed,rustdoc) + +rustdoc-clean: FORCE + $(Q)rm -rf $(rustdoc_output) + +quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< + cmd_rustc_test_library = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + @$(objtree)/include/generated/rustc_cfg \ + --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ + --out-dir $(objtree)/$(obj)/test --cfg testlib \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< + +rusttestlib-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rusttestlib-zerocopy: private rustc_target_flags = $(zerocopy-flags) +rusttestlib-zerocopy: $(src)/zerocopy/src/lib.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-build_error: $(src)/build_error.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-ffi: $(src)/ffi.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rusttestlib-proc_macro2: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-quote: private skip_flags = $(quote-skip_flags) +rusttestlib-quote: private rustc_target_flags = $(quote-flags) +rusttestlib-quote: $(src)/quote/lib.rs rusttestlib-proc_macro2 FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-syn: private rustc_target_flags = $(syn-flags) +rusttestlib-syn: $(src)/syn/lib.rs rusttestlib-quote FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro +rusttestlib-zerocopy_derive: private rustc_test_library_proc = yes +rusttestlib-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-macros: private rustc_target_flags = --extern proc_macro \ + --extern proc_macro2 --extern quote --extern syn +rusttestlib-macros: private rustc_test_library_proc = yes +rusttestlib-macros: $(src)/macros/lib.rs \ + rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \ + --extern proc_macro +rusttestlib-pin_init_internal: private rustc_test_library_proc = yes +rusttestlib-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ + rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-pin_init: private rustc_target_flags = $(pin_init-flags) +rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ + rusttestlib-pin_init_internal $(obj)/$(libpin_init_internal_name) FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-kernel: private rustc_target_flags = --extern ffi \ + --extern build_error --extern macros --extern pin_init \ + --extern bindings --extern uapi \ + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib \ + --extern zerocopy_derive=$(objtree)/$(obj)/test/$(libzerocopy_derive_name) +rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ + rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ + $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-uapi: private rustc_target_flags = --extern ffi --extern pin_init +rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test_library) + +quiet_cmd_rustdoc_test = RUSTDOC T $< + cmd_rustdoc_test = \ + $(rustc_target_envs) \ + RUST_MODFILE=test.rs \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(rust_common_flags) \ + -Zcrate-attr='feature(used_with_arg)' \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) $(rustdoc_test_target_flags) \ + $(rustdoc_test_quiet) \ + -L$(objtree)/$(obj)/test --output $(rustdoc_output) \ + --crate-name $(subst rusttest-,,$@) $< + +quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< + cmd_rustdoc_test_kernel = \ + rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ + mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(filter-out --remap-path-scope=%,$(rust_flags)) \ + -L$(objtree)/$(obj) --extern ffi --extern pin_init \ + --extern kernel --extern build_error --extern macros \ + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive \ + --no-run --crate-name kernel -Zunstable-options \ + --sysroot=/dev/null \ + $(doctests_modifiers_workaround) \ + --test-builder $(objtree)/scripts/rustdoc_test_builder \ + $< $(rustdoc_test_kernel_quiet); \ + $(objtree)/scripts/rustdoc_test_gen + +%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \ + $(src)/kernel/lib.rs $(obj)/kernel.o \ + $(objtree)/scripts/rustdoc_test_builder \ + $(objtree)/scripts/rustdoc_test_gen FORCE + +$(call if_changed,rustdoc_test_kernel) + +# We cannot use `-Zpanic-abort-tests` because some tests are dynamic, +# so for the moment we skip `-Cpanic=abort`. +quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< + cmd_rustc_test = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$@) $<; \ + $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \ + $(rustc_test_run_flags) + +rusttest: rusttest-macros + +rusttest-macros: private rustc_target_flags = --extern proc_macro \ + --extern macros --extern kernel --extern pin_init \ + --extern proc_macro2 --extern quote --extern syn +rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro +rusttest-macros: $(src)/macros/lib.rs \ + rusttestlib-macros rusttestlib-kernel rusttestlib-pin_init FORCE + +$(call if_changed,rustc_test) + +$(call if_changed,rustdoc_test) + +ifdef CONFIG_CC_IS_CLANG +bindgen_c_flags = $(c_flags) +else +# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC +# plugin backend and/or the Clang driver would be perfectly compatible with GCC. +# +# For the moment, here we are tweaking the flags on the fly. This is a hack, +# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT` +# if we end up using one of those structs). +bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ + -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \ + -mindirect-branch=thunk-extern -mindirect-branch-register \ + -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \ + -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \ + -mno-pointers-to-nested-functions -mno-string \ + -mno-strict-align -mstrict-align -mdirect-extern-access \ + -mexplicit-relocs -mno-check-zero-division \ + -fconserve-stack -falign-jumps=% -falign-loops=% \ + -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \ + -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \ + -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \ + -fzero-call-used-regs=% -fno-stack-clash-protection \ + -fno-inline-functions-called-once -fsanitize=bounds-strict \ + -fstrict-flex-arrays=% -fmin-function-alignment=% \ + -fzero-init-padding-bits=% -mno-fdpic \ + -fdiagnostics-show-context -fdiagnostics-show-context=% \ + --param=% --param asan-% -fno-isolate-erroneous-paths-dereference \ + -ffixed-r2 -mmultiple -mno-readonly-in-sdata + +# Derived from `scripts/Makefile.clang`. +BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET_arm64 := aarch64-linux-gnu +BINDGEN_TARGET_arm := arm-linux-gnueabi +BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf +BINDGEN_TARGET_s390 := s390x-linux-gnu +# This is only for i386 UM builds, which need the 32-bit target not -m32 +BINDGEN_TARGET_i386 := i386-linux-gnu + +ifdef CONFIG_PPC64 +BINDGEN_TARGET_powerpc := powerpc64le-linux-gnu +else +BINDGEN_TARGET_powerpc := powerpc-linux-gnu +endif + +BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) +BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) + +# All warnings are inhibited since GCC builds are very experimental, +# many GCC warnings are not supported by Clang, they may only appear in +# some configurations, with new GCC versions, etc. +bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET) + +# Auto variable zero-initialization requires an additional special option with +# clang that is going to be removed sometime in the future (likely in +# clang-18), so make sure to pass this option only if clang supports it +# (libclang major version < 16). +# +# https://github.com/llvm/llvm-project/issues/44842 +# https://github.com/llvm/llvm-project/blob/llvmorg-16.0.0-rc2/clang/docs/ReleaseNotes.rst#deprecated-compiler-flags +ifdef CONFIG_INIT_STACK_ALL_ZERO +libclang_maj_ver=$(shell $(BINDGEN) $(srctree)/scripts/rust_is_available_bindgen_libclang.h 2>&1 | sed -ne 's/.*clang version \([0-9]*\).*/\1/p') +ifeq ($(shell expr $(libclang_maj_ver) \< 16), 1) +bindgen_extra_c_flags += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang +endif +endif + +bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \ + $(bindgen_extra_c_flags) +endif + +ifdef CONFIG_LTO +bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags)) +else +bindgen_c_flags_lto = $(bindgen_c_flags) +endif + +# `-fno-builtin` is passed to avoid `bindgen` from using `clang` builtin +# prototypes for functions like `memcpy` -- if this flag is not passed, +# `bindgen`-generated prototypes use `c_ulong` or `c_uint` depending on +# architecture instead of generating `usize`. +bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ + +# `--rust-target` points to our minimum supported Rust version. +quiet_cmd_bindgen = BINDGEN $@ + cmd_bindgen = \ + $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ + --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ + --no-debug '.*' --enable-function-attribute-detection \ + -o $@ -- $(bindgen_c_flags_final) -DMODULE \ + $(bindgen_target_cflags) $(bindgen_target_extra) + +$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) +$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@ +$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + +$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) +$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + +# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn +# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here +# given it is `libclang`; but for consistency, future Clang changes and/or +# a potential future GCC backend for `bindgen`, we disable it too. +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \ + --blocklist-type '.*' --allowlist-var '' \ + --allowlist-function 'rust_helper_.*' +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \ + -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@ +$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE + $(call if_changed_dep,bindgen) + +quiet_cmd_rust_helper = HELPER $@ + cmd_rust_helper = \ + $(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) \ + -c -g0 $< -emit-llvm -o $@ + +$(obj)/helpers/helpers.bc: private part-of-builtin := y +$(obj)/helpers/helpers_module.bc: private part-of-module := y +$(obj)/helpers/helpers.bc $(obj)/helpers/helpers_module.bc: $(src)/helpers/helpers.c FORCE + +$(call if_changed_dep,rust_helper) + +rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' + +quiet_cmd_exports = EXPORTS $@ + cmd_exports = \ + $(call rust_exports,$<,"EXPORT_SYMBOL_RUST_GPL(%s);\n") > $@ + +$(obj)/exports_core_generated.h: $(obj)/core.o FORCE + $(call if_changed,exports) + +# Even though Rust kernel modules should never use the bindings directly, +# symbols from the `bindings` crate and the C helpers need to be exported +# because Rust generics and inlined functions may not get their code generated +# in the crate where they are defined. Other helpers, called from non-inline +# functions, may not be exported, in principle. However, in general, the Rust +# compiler does not guarantee codegen will be performed for a non-inline +# function either. Therefore, we export all symbols from helpers and bindings. +# In the future, this may be revisited to reduce the number of exports after +# the compiler is informed about the places codegen is required. +$(obj)/exports_helpers_generated.h: $(obj)/helpers/helpers.o FORCE + $(call if_changed,exports) + +$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE + $(call if_changed,exports) + +$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE + $(call if_changed,exports) + +quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@ + cmd_rustc_procmacrolibrary = \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ + --out-dir $(objtree)/$(obj)/host -L$(objtree)/$(obj)/host \ + --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $< + +$(obj)/host/libproc_macro2.rlib: private skip_clippy = 1 +$(obj)/host/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) +$(obj)/host/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +$(obj)/host/libquote.rlib: private skip_clippy = 1 +$(obj)/host/libquote.rlib: private skip_flags = $(quote-skip_flags) +$(obj)/host/libquote.rlib: private rustc_target_flags = $(quote-flags) +$(obj)/host/libquote.rlib: $(src)/quote/lib.rs $(obj)/host/libproc_macro2.rlib FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +$(obj)/host/libsyn.rlib: private skip_clippy = 1 +$(obj)/host/libsyn.rlib: private rustc_target_flags = $(syn-flags) +$(obj)/host/libsyn.rlib: $(src)/syn/lib.rs $(obj)/host/libquote.rlib FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + +quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@ + cmd_rustc_procmacro = \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) $(rust_common_flags) $(rustc_target_flags) \ + -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ + -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ + --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ + --crate-type proc-macro -L$(objtree)/$(obj)/host \ + --crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \ + @$(objtree)/include/generated/rustc_cfg $< + +# Procedural macros can only be used with the `rustc` that compiled it. +$(obj)/$(libzerocopy_derive_name): private skip_clippy = 1 +$(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags) +$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +$(obj)/$(libmacros_name): private rustc_target_flags = \ + --extern proc_macro2 --extern quote --extern syn +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +$(obj)/$(libpin_init_internal_name): private rustc_target_flags = $(pin_init_internal-flags) +$(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \ + $(obj)/host/libproc_macro2.rlib $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + +# `rustc` requires `-Zunstable-options` to use custom target specifications +# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). +quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ + cmd_rustc_library = \ + $(rustc_target_envs) \ + OBJTREE=$(abspath $(objtree)) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ + --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \ + --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ + --crate-type rlib -L$(objtree)/$(obj) \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ + --sysroot=/dev/null \ + -Zunstable-options \ + $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ + $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ + $(cmd_objtool) + +rust-analyzer: + $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ + --cfgs='core=$(core-cfgs)' $(core-edition) \ + --cfgs='zerocopy=$(zerocopy-cfgs)' \ + --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ + --cfgs='quote=$(quote-cfgs)' \ + --cfgs='syn=$(syn-cfgs)' \ + --cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \ + --cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \ + --cfgs='pin_init=$(pin_init-cfgs)' \ + --envs='zerocopy=$(zerocopy-envs)' \ + $(realpath $(srctree)) $(realpath $(objtree)) \ + $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ + > rust-project.json + +redirect-intrinsics = \ + __addsf3 __eqsf2 __extendsfdf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __truncdfsf2 __unordsf2 \ + __adddf3 __eqdf2 __ledf2 __ltdf2 __muldf3 __unorddf2 \ + __muloti4 __multi3 \ + __udivmodti4 __udivti3 __umodti3 + +ifdef CONFIG_ARM + # Add eabi initrinsics for ARM 32-bit + redirect-intrinsics += \ + __aeabi_fadd __aeabi_fmul __aeabi_fcmpeq __aeabi_fcmple __aeabi_fcmplt __aeabi_fcmpun \ + __aeabi_dadd __aeabi_dmul __aeabi_dcmple __aeabi_dcmplt __aeabi_dcmpun \ + __aeabi_uldivmod +endif +ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) + # These intrinsics are defined for ARM64 and RISCV64 + redirect-intrinsics += \ + __ashrti3 \ + __ashlti3 __lshrti3 +endif +ifdef CONFIG_PPC32 + redirect-intrinsics += \ + __udivdi3 __umoddi3 +endif + +ifdef CONFIG_MODVERSIONS +cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \ + $(call rust_exports,$@,"%s\n") | \ + scripts/gendwarfksyms/gendwarfksyms \ + $(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \ + $(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \ + $@ >> $(dot-target).cmd) +endif + +define rule_rustc_library + $(call cmd_and_fixdep,rustc_library) + $(call cmd,gen_objtooldep) + $(call cmd,gendwarfksyms) +endef + +define rule_rust_cc_library + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + $(call cmd,gendwarfksyms) +endef + +# helpers.o uses the same export mechanism as Rust libraries, so ensure symbol +# versions are calculated for the helpers too. +$(obj)/helpers/helpers.o: $(src)/helpers/helpers.c $(recordmcount_source) FORCE + +$(call if_changed_rule,rust_cc_library) + +# Disable symbol versioning for exports.o to avoid conflicts with the actual +# symbol versions generated from Rust objects. +$(obj)/exports.o: private skip_gendwarfksyms = 1 + +$(obj)/core.o: private skip_clippy = 1 +$(obj)/core.o: private skip_flags = $(core-skip_flags) +$(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) +$(obj)/core.o: private rustc_target_flags = $(core-flags) +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \ + $(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE + +$(call if_changed_rule,rustc_library) +ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),) +$(obj)/core.o: scripts/target.json +endif +KCOV_INSTRUMENT_core.o := n + +$(obj)/compiler_builtins.o: private skip_gendwarfksyms = 1 +$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' +$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/zerocopy.o: private skip_clippy = 1 +$(obj)/zerocopy.o: private skip_gendwarfksyms = 1 +$(obj)/zerocopy.o: private rustc_target_envs := $(zerocopy-envs) +$(obj)/zerocopy.o: private rustc_target_flags = $(zerocopy-flags) +$(obj)/zerocopy.o: $(src)/zerocopy/src/lib.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/pin_init.o: private skip_gendwarfksyms = 1 +$(obj)/pin_init.o: private rustc_target_flags = $(pin_init-flags) +$(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \ + $(obj)/$(libpin_init_internal_name) $(obj)/$(libmacros_name) FORCE + +$(call if_changed_rule,rustc_library) + +# Even if normally `build_error` is not a kernel object, it should still be +# treated as such so that we pass the same flags. Otherwise, for instance, +# `rustc` will complain about missing sanitizer flags causing an ABI mismatch. +$(obj)/build_error.o: private is-kernel-object := y +$(obj)/build_error.o: private skip_gendwarfksyms = 1 +$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/ffi.o: private skip_gendwarfksyms = 1 +$(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/bindings.o: private rustc_target_flags = --extern ffi --extern pin_init +$(obj)/bindings.o: $(src)/bindings/lib.rs \ + $(obj)/ffi.o \ + $(obj)/pin_init.o \ + $(obj)/bindings/bindings_generated.rs \ + $(obj)/bindings/bindings_helpers_generated.rs FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/uapi.o: private rustc_target_flags = --extern ffi --extern pin_init +$(obj)/uapi.o: private skip_gendwarfksyms = 1 +$(obj)/uapi.o: $(src)/uapi/lib.rs \ + $(obj)/ffi.o \ + $(obj)/pin_init.o \ + $(obj)/uapi/uapi_generated.rs FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \ + --extern build_error --extern macros --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ + $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o \ + $(obj)/zerocopy.o $(obj)/$(libzerocopy_derive_name) FORCE + +$(call if_changed_rule,rustc_library) + +ifdef CONFIG_JUMP_LABEL +$(obj)/kernel.o: $(obj)/kernel/generated_arch_static_branch_asm.rs +endif +ifndef CONFIG_UML +ifdef CONFIG_BUG +$(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generated_arch_reachable_asm.rs +endif +endif + +ifdef CONFIG_RUST_INLINE_HELPERS +$(obj)/kernel.o: private link_helper = 1 +$(obj)/kernel.o: $(obj)/helpers/helpers.bc +endif + +endif # CONFIG_RUST diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters new file mode 100644 index 000000000..8402b0c93 --- /dev/null +++ b/rust/bindgen_parameters @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: GPL-2.0 + +# We want to map these types to `isize`/`usize` manually, instead of +# define them as `int`/`long` depending on platform bitwidth. +--blocklist-type __kernel_s?size_t +--blocklist-type __kernel_ptrdiff_t + +--opaque-type xregs_state +--opaque-type desc_struct +--opaque-type arch_lbr_state +--opaque-type local_apic + +# Packed type cannot transitively contain a `#[repr(align)]` type. +--opaque-type alt_instr +--opaque-type x86_msi_data +--opaque-type x86_msi_addr_lo +# s390-only: same packed/align issue as above (E0588). +--opaque-type lowcore +--opaque-type tod_clock +--opaque-type tpi_info +--opaque-type uv_cb.* +--opaque-type uv_secret.* +--opaque-type zpci_fib + +# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust +# warning. We don't need to peek into it anyway. +--opaque-type spinlock + +# enums that appear in indirect function calls should specify a cfi type +--newtype-enum lru_status +--with-attribute-custom-enum=lru_status='#[cfi_encoding="10lru_status"]' + +# `seccomp`'s comment gets understood as a doctest +--no-doc-comments + +# These functions use the `__preserve_most` calling convention, which neither bindgen +# nor Rust currently understand, and which Clang currently declares to be unstable. +--blocklist-function __list_.*_report + +# These constants are sometimes not recognized by bindgen depending on config. +# We use const helpers to aid bindgen, to avoid conflicts when constants are +# recognized, block generation of the non-helper constants. +--blocklist-item ARCH_SLAB_MINALIGN +--blocklist-item ARCH_KMALLOC_MINALIGN +--blocklist-item VM_MERGEABLE +--blocklist-item VM_READ +--blocklist-item VM_WRITE +--blocklist-item VM_EXEC +--blocklist-item VM_SHARED +--blocklist-item VM_MAYREAD +--blocklist-item VM_MAYWRITE +--blocklist-item VM_MAYEXEC +--blocklist-item VM_MAYEXEC +--blocklist-item VM_PFNMAP +--blocklist-item VM_IO +--blocklist-item VM_DONTCOPY +--blocklist-item VM_DONTEXPAND +--blocklist-item VM_LOCKONFAULT +--blocklist-item VM_ACCOUNT +--blocklist-item VM_NORESERVE +--blocklist-item VM_HUGETLB +--blocklist-item VM_SYNC +--blocklist-item VM_ARCH_1 +--blocklist-item VM_WIPEONFORK +--blocklist-item VM_DONTDUMP +--blocklist-item VM_SOFTDIRTY +--blocklist-item VM_MIXEDMAP +--blocklist-item VM_HUGEPAGE +--blocklist-item VM_NOHUGEPAGE + +# Structs should implement `Zeroable` when all of their fields do. +--with-derive-custom-struct .*=MaybeZeroable +--with-derive-custom-union .*=MaybeZeroable diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h new file mode 100644 index 000000000..4b31aa7f4 --- /dev/null +++ b/rust/bindings/bindings_helper.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Header that contains the code (mostly headers) for which Rust bindings + * will be automatically generated by `bindgen`. + * + * Sorted alphabetically. + */ + +/* + * First, avoid forward references to `enum` types. + * + * This workarounds a `bindgen` issue with them: + * . + * + * Without this, the generated Rust type may be the wrong one (`i32`) or + * the proper one (typically `c_uint`) depending on how the headers are + * included, which in turn may depend on the particular kernel configuration + * or the architecture. + * + * The alternative would be to use casts and likely an + * `#[allow(clippy::unnecessary_cast)]` in the Rust source files. Instead, + * this approach allows us to keep the correct code in the source files and + * simply remove this section when the issue is fixed upstream and we bump + * the minimum `bindgen` version. + * + * This workaround may not be possible in some cases, depending on how the C + * headers are set up. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * The driver-core Rust code needs to know about some C driver-core private + * structures. + */ +#include <../../drivers/base/base.h> + +#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE) +// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`. +#include +#endif + +/* `bindgen` gets confused at certain things. */ +const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; +const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; +const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; +const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; +const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; +const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; +const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; +const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; +const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; + +const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; + +const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; +const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC1 = XA_FLAGS_ALLOC1; + +const vm_flags_t RUST_CONST_HELPER_VM_MERGEABLE = VM_MERGEABLE; +const vm_flags_t RUST_CONST_HELPER_VM_READ = VM_READ; +const vm_flags_t RUST_CONST_HELPER_VM_WRITE = VM_WRITE; +const vm_flags_t RUST_CONST_HELPER_VM_EXEC = VM_EXEC; +const vm_flags_t RUST_CONST_HELPER_VM_SHARED = VM_SHARED; +const vm_flags_t RUST_CONST_HELPER_VM_MAYREAD = VM_MAYREAD; +const vm_flags_t RUST_CONST_HELPER_VM_MAYWRITE = VM_MAYWRITE; +const vm_flags_t RUST_CONST_HELPER_VM_MAYEXEC = VM_MAYEXEC; +const vm_flags_t RUST_CONST_HELPER_VM_MAYSHARE = VM_MAYEXEC; +const vm_flags_t RUST_CONST_HELPER_VM_PFNMAP = VM_PFNMAP; +const vm_flags_t RUST_CONST_HELPER_VM_IO = VM_IO; +const vm_flags_t RUST_CONST_HELPER_VM_DONTCOPY = VM_DONTCOPY; +const vm_flags_t RUST_CONST_HELPER_VM_DONTEXPAND = VM_DONTEXPAND; +const vm_flags_t RUST_CONST_HELPER_VM_LOCKONFAULT = VM_LOCKONFAULT; +const vm_flags_t RUST_CONST_HELPER_VM_ACCOUNT = VM_ACCOUNT; +const vm_flags_t RUST_CONST_HELPER_VM_NORESERVE = VM_NORESERVE; +const vm_flags_t RUST_CONST_HELPER_VM_HUGETLB = VM_HUGETLB; +const vm_flags_t RUST_CONST_HELPER_VM_SYNC = VM_SYNC; +const vm_flags_t RUST_CONST_HELPER_VM_ARCH_1 = VM_ARCH_1; +const vm_flags_t RUST_CONST_HELPER_VM_WIPEONFORK = VM_WIPEONFORK; +const vm_flags_t RUST_CONST_HELPER_VM_DONTDUMP = VM_DONTDUMP; +const vm_flags_t RUST_CONST_HELPER_VM_SOFTDIRTY = VM_SOFTDIRTY; +const vm_flags_t RUST_CONST_HELPER_VM_MIXEDMAP = VM_MIXEDMAP; +const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE; +const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE; + +#if IS_ENABLED(CONFIG_GPU_BUDDY) +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TOPDOWN_ALLOCATION = GPU_BUDDY_TOPDOWN_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CONTIGUOUS_ALLOCATION = + GPU_BUDDY_CONTIGUOUS_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEAR_ALLOCATION = GPU_BUDDY_CLEAR_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; +#endif + +#if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) +#include "../../drivers/android/binder/rust_binder.h" +#include "../../drivers/android/binder/rust_binder_events.h" +#endif diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs new file mode 100644 index 000000000..439ab88a5 --- /dev/null +++ b/rust/bindings/lib.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bindings. +//! +//! Imports the generated bindings by `bindgen`. +//! +//! This crate may not be directly used. If you need a kernel C API that is +//! not ported or wrapped in the `kernel` crate, then do so first instead of +//! using this crate. + +#![no_std] +#![allow( + clippy::all, + missing_docs, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + improper_ctypes, + unreachable_pub, + unsafe_op_in_unsafe_fn +)] +#![feature(cfi_encoding)] + +#[allow(dead_code)] +#[allow(clippy::as_underscore)] +#[allow(clippy::cast_lossless)] +#[allow(clippy::ptr_as_ptr)] +#[allow(clippy::ref_as_ptr)] +#[allow(clippy::undocumented_unsafe_blocks)] +#[cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#[allow(unnecessary_transmutes)] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] +mod bindings_raw { + use pin_init::{MaybeZeroable, Zeroable}; + + // Manual definition for blocklisted types. + type __kernel_size_t = usize; + type __kernel_ssize_t = isize; + type __kernel_ptrdiff_t = isize; + + // `bindgen` doesn't automatically do this, see + // + // + // SAFETY: `__BindgenBitfieldUnit` is a newtype around `Storage`. + unsafe impl Zeroable for __BindgenBitfieldUnit where Storage: Zeroable {} + + // Use glob import here to expose all helpers. + // Symbols defined within the module will take precedence to the glob import. + pub use super::bindings_helper::*; + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_generated.rs" + )); +} + +// When both a directly exposed symbol and a helper exists for the same function, +// the directly exposed symbol is preferred and the helper becomes dead code, so +// ignore the warning here. +#[allow(dead_code)] +mod bindings_helper { + // Import the generated bindings for types. + use super::bindings_raw::*; + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_helpers_generated.rs" + )); +} + +pub use bindings_raw::*; + +pub const compat_ptr_ioctl: Option< + unsafe extern "C" fn(*mut file, ffi::c_uint, ffi::c_ulong) -> ffi::c_long, +> = { + #[cfg(CONFIG_COMPAT)] + { + Some(bindings_raw::compat_ptr_ioctl) + } + #[cfg(not(CONFIG_COMPAT))] + { + None + } +}; diff --git a/rust/build_error.rs b/rust/build_error.rs new file mode 100644 index 000000000..fa24eeef9 --- /dev/null +++ b/rust/build_error.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Build-time error. +//! +//! This crate provides a [const function][const-functions] `build_error`, which will panic in +//! compile-time if executed in [const context][const-context], and will cause a build error +//! if not executed at compile time and the optimizer does not optimise away the call. +//! +//! It is used by `build_assert!` in the kernel crate, allowing checking of +//! conditions that could be checked statically, but could not be enforced in +//! Rust yet (e.g. perform some checks in [const functions][const-functions], but those +//! functions could still be called in the runtime). +//! +//! For details on constant evaluation in Rust, please see the [Reference][const-eval]. +//! +//! [const-eval]: https://doc.rust-lang.org/reference/const_eval.html +//! [const-functions]: https://doc.rust-lang.org/reference/const_eval.html#const-functions +//! [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context + +#![no_std] + +/// Panics if executed in [const context][const-context], or triggers a build error if not. +/// +/// [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context +#[inline(never)] +#[cold] +#[export_name = "rust_build_error"] +#[track_caller] +pub const fn build_error(msg: &'static str) -> ! { + panic!("{}", msg); +} diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs new file mode 100644 index 000000000..fc6b54636 --- /dev/null +++ b/rust/compiler_builtins.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Our own `compiler_builtins`. +//! +//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`]. +//! Since we do not need the vast majority of them, we avoid the dependency +//! by providing this file. +//! +//! At the moment, some builtins are required that should not be. For instance, +//! [`core`] has 128-bit integers functionality which we should not be compiling +//! in. We will work with upstream [`core`] to provide feature flags to disable +//! the parts we do not need. For the moment, we define them to [`panic!`] at +//! runtime for simplicity to catch mistakes, instead of performing surgery +//! on `core.o`. +//! +//! In any case, all these symbols are weakened to ensure we do not override +//! those that may be provided by the rest of the kernel. +//! +//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins +//! [`compiler-rt`]: https://compiler-rt.llvm.org/ + +#![allow(internal_features)] +#![feature(compiler_builtins)] +#![compiler_builtins] +#![no_builtins] +#![no_std] + +macro_rules! define_panicking_intrinsics( + ($reason: tt, { $($ident: ident, )* }) => { + $( + #[doc(hidden)] + #[export_name = concat!("__rust", stringify!($ident))] + pub extern "C" fn $ident() { + panic!($reason); + } + )* + } +); + +define_panicking_intrinsics!("`f32` should not be used", { + __addsf3, + __eqsf2, + __extendsfdf2, + __gesf2, + __lesf2, + __ltsf2, + __mulsf3, + __nesf2, + __truncdfsf2, + __unordsf2, +}); + +define_panicking_intrinsics!("`f64` should not be used", { + __adddf3, + __eqdf2, + __ledf2, + __ltdf2, + __muldf3, + __unorddf2, +}); + +define_panicking_intrinsics!("`i128` should not be used", { + __ashrti3, + __muloti4, + __multi3, +}); + +define_panicking_intrinsics!("`u128` should not be used", { + __ashlti3, + __lshrti3, + __udivmodti4, + __udivti3, + __umodti3, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f32` should not be used", { + __aeabi_fadd, + __aeabi_fmul, + __aeabi_fcmpeq, + __aeabi_fcmple, + __aeabi_fcmplt, + __aeabi_fcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f64` should not be used", { + __aeabi_dadd, + __aeabi_dmul, + __aeabi_dcmple, + __aeabi_dcmplt, + __aeabi_dcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __aeabi_uldivmod, +}); + +#[cfg(target_arch = "powerpc")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __udivdi3, + __umoddi3, +}); + +// NOTE: if you are adding a new intrinsic here, you should also add it to +// `redirect-intrinsics` in `rust/Makefile`. diff --git a/rust/exports.c b/rust/exports.c new file mode 100644 index 000000000..1b52460b0 --- /dev/null +++ b/rust/exports.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A hack to export Rust symbols for loadable modules without having to redo + * the entire `include/linux/export.h` logic in Rust. + * + * This requires Rust's new/future `v0` mangling scheme because the default one + * ("legacy") uses invalid characters for C identifiers (thus we cannot use the + * `EXPORT_SYMBOL_*` macros). + * + * All symbols are exported as GPL-only to guarantee no GPL-only feature is + * accidentally exposed. + */ + +#include + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) + +#include "exports_core_generated.h" +#include "exports_bindings_generated.h" +#include "exports_kernel_generated.h" + +#ifndef CONFIG_RUST_INLINE_HELPERS +#include "exports_helpers_generated.h" +#endif + +// For modules using `rust/build_error.rs`. +#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW +EXPORT_SYMBOL_RUST_GPL(rust_build_error); +#endif diff --git a/rust/ffi.rs b/rust/ffi.rs new file mode 100644 index 000000000..f961e9728 --- /dev/null +++ b/rust/ffi.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Foreign function interface (FFI) types. +//! +//! This crate provides mapping from C primitive types to Rust ones. +//! +//! The Rust [`core`] crate provides [`core::ffi`], which maps integer types to the platform default +//! C ABI. The kernel does not use [`core::ffi`], so it can customise the mapping that deviates from +//! the platform default. + +#![no_std] + +macro_rules! alias { + ($($name:ident = $ty:ty;)*) => {$( + #[allow(non_camel_case_types, missing_docs)] + pub type $name = $ty; + + // Check size compatibility with `core`. + const _: () = assert!( + ::core::mem::size_of::<$name>() == ::core::mem::size_of::<::core::ffi::$name>() + ); + )*} +} + +alias! { + // `core::ffi::c_char` is either `i8` or `u8` depending on architecture. In the kernel, we use + // `-funsigned-char` so it's always mapped to `u8`. + c_char = u8; + + c_schar = i8; + c_uchar = u8; + + c_short = i16; + c_ushort = u16; + + c_int = i32; + c_uint = u32; + + // In the kernel, `intptr_t` is defined to be `long` in all platforms, so we can map the type to + // `isize`. + c_long = isize; + c_ulong = usize; + + c_longlong = i64; + c_ulonglong = u64; +} + +pub use core::ffi::c_void; + +pub use core::ffi::CStr; diff --git a/rust/helpers/atomic.c b/rust/helpers/atomic.c new file mode 100644 index 000000000..4b24eceef --- /dev/null +++ b/rust/helpers/atomic.c @@ -0,0 +1,1035 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-rust-atomic-helpers.sh +// DO NOT MODIFY THIS FILE DIRECTLY + +/* + * This file provides helpers for the various atomic functions for Rust. + */ +#ifndef _RUST_ATOMIC_API_H +#define _RUST_ATOMIC_API_H + +#include + +__rust_helper int +rust_helper_atomic_read(const atomic_t *v) +{ + return atomic_read(v); +} + +__rust_helper int +rust_helper_atomic_read_acquire(const atomic_t *v) +{ + return atomic_read_acquire(v); +} + +__rust_helper void +rust_helper_atomic_set(atomic_t *v, int i) +{ + atomic_set(v, i); +} + +__rust_helper void +rust_helper_atomic_set_release(atomic_t *v, int i) +{ + atomic_set_release(v, i); +} + +__rust_helper void +rust_helper_atomic_add(int i, atomic_t *v) +{ + atomic_add(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return(int i, atomic_t *v) +{ + return atomic_add_return(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_acquire(int i, atomic_t *v) +{ + return atomic_add_return_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_release(int i, atomic_t *v) +{ + return atomic_add_return_release(i, v); +} + +__rust_helper int +rust_helper_atomic_add_return_relaxed(int i, atomic_t *v) +{ + return atomic_add_return_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add(int i, atomic_t *v) +{ + return atomic_fetch_add(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_acquire(int i, atomic_t *v) +{ + return atomic_fetch_add_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_release(int i, atomic_t *v) +{ + return atomic_fetch_add_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_add_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_sub(int i, atomic_t *v) +{ + atomic_sub(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return(int i, atomic_t *v) +{ + return atomic_sub_return(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_acquire(int i, atomic_t *v) +{ + return atomic_sub_return_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_release(int i, atomic_t *v) +{ + return atomic_sub_return_release(i, v); +} + +__rust_helper int +rust_helper_atomic_sub_return_relaxed(int i, atomic_t *v) +{ + return atomic_sub_return_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub(int i, atomic_t *v) +{ + return atomic_fetch_sub(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_acquire(int i, atomic_t *v) +{ + return atomic_fetch_sub_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_release(int i, atomic_t *v) +{ + return atomic_fetch_sub_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_sub_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_sub_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_inc(atomic_t *v) +{ + atomic_inc(v); +} + +__rust_helper int +rust_helper_atomic_inc_return(atomic_t *v) +{ + return atomic_inc_return(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_acquire(atomic_t *v) +{ + return atomic_inc_return_acquire(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_release(atomic_t *v) +{ + return atomic_inc_return_release(v); +} + +__rust_helper int +rust_helper_atomic_inc_return_relaxed(atomic_t *v) +{ + return atomic_inc_return_relaxed(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc(atomic_t *v) +{ + return atomic_fetch_inc(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_acquire(atomic_t *v) +{ + return atomic_fetch_inc_acquire(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_release(atomic_t *v) +{ + return atomic_fetch_inc_release(v); +} + +__rust_helper int +rust_helper_atomic_fetch_inc_relaxed(atomic_t *v) +{ + return atomic_fetch_inc_relaxed(v); +} + +__rust_helper void +rust_helper_atomic_dec(atomic_t *v) +{ + atomic_dec(v); +} + +__rust_helper int +rust_helper_atomic_dec_return(atomic_t *v) +{ + return atomic_dec_return(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_acquire(atomic_t *v) +{ + return atomic_dec_return_acquire(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_release(atomic_t *v) +{ + return atomic_dec_return_release(v); +} + +__rust_helper int +rust_helper_atomic_dec_return_relaxed(atomic_t *v) +{ + return atomic_dec_return_relaxed(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec(atomic_t *v) +{ + return atomic_fetch_dec(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_acquire(atomic_t *v) +{ + return atomic_fetch_dec_acquire(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_release(atomic_t *v) +{ + return atomic_fetch_dec_release(v); +} + +__rust_helper int +rust_helper_atomic_fetch_dec_relaxed(atomic_t *v) +{ + return atomic_fetch_dec_relaxed(v); +} + +__rust_helper void +rust_helper_atomic_and(int i, atomic_t *v) +{ + atomic_and(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and(int i, atomic_t *v) +{ + return atomic_fetch_and(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_acquire(int i, atomic_t *v) +{ + return atomic_fetch_and_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_release(int i, atomic_t *v) +{ + return atomic_fetch_and_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_and_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_and_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_andnot(int i, atomic_t *v) +{ + atomic_andnot(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot(int i, atomic_t *v) +{ + return atomic_fetch_andnot(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + return atomic_fetch_andnot_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_release(int i, atomic_t *v) +{ + return atomic_fetch_andnot_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_andnot_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_andnot_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_or(int i, atomic_t *v) +{ + atomic_or(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or(int i, atomic_t *v) +{ + return atomic_fetch_or(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_acquire(int i, atomic_t *v) +{ + return atomic_fetch_or_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_release(int i, atomic_t *v) +{ + return atomic_fetch_or_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_or_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_or_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic_xor(int i, atomic_t *v) +{ + atomic_xor(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor(int i, atomic_t *v) +{ + return atomic_fetch_xor(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_acquire(int i, atomic_t *v) +{ + return atomic_fetch_xor_acquire(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_release(int i, atomic_t *v) +{ + return atomic_fetch_xor_release(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_xor_relaxed(int i, atomic_t *v) +{ + return atomic_fetch_xor_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_xchg(atomic_t *v, int new) +{ + return atomic_xchg(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_acquire(atomic_t *v, int new) +{ + return atomic_xchg_acquire(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_release(atomic_t *v, int new) +{ + return atomic_xchg_release(v, new); +} + +__rust_helper int +rust_helper_atomic_xchg_relaxed(atomic_t *v, int new) +{ + return atomic_xchg_relaxed(v, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_acquire(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_release(v, old, new); +} + +__rust_helper int +rust_helper_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +{ + return atomic_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_acquire(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_release(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +{ + return atomic_try_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic_sub_and_test(int i, atomic_t *v) +{ + return atomic_sub_and_test(i, v); +} + +__rust_helper bool +rust_helper_atomic_dec_and_test(atomic_t *v) +{ + return atomic_dec_and_test(v); +} + +__rust_helper bool +rust_helper_atomic_inc_and_test(atomic_t *v) +{ + return atomic_inc_and_test(v); +} + +__rust_helper bool +rust_helper_atomic_add_negative(int i, atomic_t *v) +{ + return atomic_add_negative(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_acquire(int i, atomic_t *v) +{ + return atomic_add_negative_acquire(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_release(int i, atomic_t *v) +{ + return atomic_add_negative_release(i, v); +} + +__rust_helper bool +rust_helper_atomic_add_negative_relaxed(int i, atomic_t *v) +{ + return atomic_add_negative_relaxed(i, v); +} + +__rust_helper int +rust_helper_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + return atomic_fetch_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic_add_unless(atomic_t *v, int a, int u) +{ + return atomic_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic_inc_not_zero(atomic_t *v) +{ + return atomic_inc_not_zero(v); +} + +__rust_helper bool +rust_helper_atomic_inc_unless_negative(atomic_t *v) +{ + return atomic_inc_unless_negative(v); +} + +__rust_helper bool +rust_helper_atomic_dec_unless_positive(atomic_t *v) +{ + return atomic_dec_unless_positive(v); +} + +__rust_helper int +rust_helper_atomic_dec_if_positive(atomic_t *v) +{ + return atomic_dec_if_positive(v); +} + +__rust_helper s64 +rust_helper_atomic64_read(const atomic64_t *v) +{ + return atomic64_read(v); +} + +__rust_helper s64 +rust_helper_atomic64_read_acquire(const atomic64_t *v) +{ + return atomic64_read_acquire(v); +} + +__rust_helper void +rust_helper_atomic64_set(atomic64_t *v, s64 i) +{ + atomic64_set(v, i); +} + +__rust_helper void +rust_helper_atomic64_set_release(atomic64_t *v, s64 i) +{ + atomic64_set_release(v, i); +} + +__rust_helper void +rust_helper_atomic64_add(s64 i, atomic64_t *v) +{ + atomic64_add(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return(s64 i, atomic64_t *v) +{ + return atomic64_add_return(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_acquire(s64 i, atomic64_t *v) +{ + return atomic64_add_return_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_release(s64 i, atomic64_t *v) +{ + return atomic64_add_return_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_add_return_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_add_return_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_add_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_sub(s64 i, atomic64_t *v) +{ + atomic64_sub(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return(s64 i, atomic64_t *v) +{ + return atomic64_sub_return(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_acquire(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_release(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_sub_return_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_sub_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_inc(atomic64_t *v) +{ + atomic64_inc(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return(atomic64_t *v) +{ + return atomic64_inc_return(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_acquire(atomic64_t *v) +{ + return atomic64_inc_return_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_release(atomic64_t *v) +{ + return atomic64_inc_return_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_inc_return_relaxed(atomic64_t *v) +{ + return atomic64_inc_return_relaxed(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc(atomic64_t *v) +{ + return atomic64_fetch_inc(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + return atomic64_fetch_inc_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_release(atomic64_t *v) +{ + return atomic64_fetch_inc_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_inc_relaxed(atomic64_t *v) +{ + return atomic64_fetch_inc_relaxed(v); +} + +__rust_helper void +rust_helper_atomic64_dec(atomic64_t *v) +{ + atomic64_dec(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return(atomic64_t *v) +{ + return atomic64_dec_return(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_acquire(atomic64_t *v) +{ + return atomic64_dec_return_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_release(atomic64_t *v) +{ + return atomic64_dec_return_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_return_relaxed(atomic64_t *v) +{ + return atomic64_dec_return_relaxed(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec(atomic64_t *v) +{ + return atomic64_fetch_dec(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_acquire(atomic64_t *v) +{ + return atomic64_fetch_dec_acquire(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_release(atomic64_t *v) +{ + return atomic64_fetch_dec_release(v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_dec_relaxed(atomic64_t *v) +{ + return atomic64_fetch_dec_relaxed(v); +} + +__rust_helper void +rust_helper_atomic64_and(s64 i, atomic64_t *v) +{ + atomic64_and(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_and_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_andnot(s64 i, atomic64_t *v) +{ + atomic64_andnot(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_andnot_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_or(s64 i, atomic64_t *v) +{ + atomic64_or(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_or_relaxed(i, v); +} + +__rust_helper void +rust_helper_atomic64_xor(s64 i, atomic64_t *v) +{ + atomic64_xor(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_acquire(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_release(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_release(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_fetch_xor_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_xchg(atomic64_t *v, s64 new) +{ + return atomic64_xchg(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_acquire(atomic64_t *v, s64 new) +{ + return atomic64_xchg_acquire(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_release(atomic64_t *v, s64 new) +{ + return atomic64_xchg_release(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_xchg_relaxed(atomic64_t *v, s64 new) +{ + return atomic64_xchg_relaxed(v, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_acquire(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_release(v, old, new); +} + +__rust_helper s64 +rust_helper_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + return atomic64_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_acquire(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_release(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +{ + return atomic64_try_cmpxchg_relaxed(v, old, new); +} + +__rust_helper bool +rust_helper_atomic64_sub_and_test(s64 i, atomic64_t *v) +{ + return atomic64_sub_and_test(i, v); +} + +__rust_helper bool +rust_helper_atomic64_dec_and_test(atomic64_t *v) +{ + return atomic64_dec_and_test(v); +} + +__rust_helper bool +rust_helper_atomic64_inc_and_test(atomic64_t *v) +{ + return atomic64_inc_and_test(v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative(s64 i, atomic64_t *v) +{ + return atomic64_add_negative(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_acquire(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_acquire(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_release(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_release(i, v); +} + +__rust_helper bool +rust_helper_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) +{ + return atomic64_add_negative_relaxed(i, v); +} + +__rust_helper s64 +rust_helper_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return atomic64_fetch_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return atomic64_add_unless(v, a, u); +} + +__rust_helper bool +rust_helper_atomic64_inc_not_zero(atomic64_t *v) +{ + return atomic64_inc_not_zero(v); +} + +__rust_helper bool +rust_helper_atomic64_inc_unless_negative(atomic64_t *v) +{ + return atomic64_inc_unless_negative(v); +} + +__rust_helper bool +rust_helper_atomic64_dec_unless_positive(atomic64_t *v) +{ + return atomic64_dec_unless_positive(v); +} + +__rust_helper s64 +rust_helper_atomic64_dec_if_positive(atomic64_t *v) +{ + return atomic64_dec_if_positive(v); +} + +#endif /* _RUST_ATOMIC_API_H */ +// e4edb6174dd42a265284958f00a7cea7ddb464b1 diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c new file mode 100644 index 000000000..c267d5190 --- /dev/null +++ b/rust/helpers/atomic_ext.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#define GEN_READ_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read(type *ptr) \ +{ \ + return READ_ONCE(*ptr); \ +} + +#define GEN_SET_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set(type *ptr, type val) \ +{ \ + WRITE_ONCE(*ptr, val); \ +} + +#define GEN_READ_ACQUIRE_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read_acquire(type *ptr) \ +{ \ + return smp_load_acquire(ptr); \ +} + +#define GEN_SET_RELEASE_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set_release(type *ptr, type val)\ +{ \ + smp_store_release(ptr, val); \ +} + +#define GEN_READ_SET_HELPERS(tname, type) \ + GEN_READ_HELPER(tname, type) \ + GEN_SET_HELPER(tname, type) \ + GEN_READ_ACQUIRE_HELPER(tname, type) \ + GEN_SET_RELEASE_HELPER(tname, type) \ + +GEN_READ_SET_HELPERS(i8, s8) +GEN_READ_SET_HELPERS(i16, s16) +GEN_READ_SET_HELPERS(ptr, const void *) + +/* + * xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding xchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +#define GEN_XCHG_HELPER(tname, type, suffix) \ +__rust_helper type \ +rust_helper_atomic_##tname##_xchg##suffix(type *ptr, type new) \ +{ \ + return xchg##suffix(ptr, new); \ +} + +#define GEN_XCHG_HELPERS(tname, type) \ + GEN_XCHG_HELPER(tname, type, ) \ + GEN_XCHG_HELPER(tname, type, _acquire) \ + GEN_XCHG_HELPER(tname, type, _release) \ + GEN_XCHG_HELPER(tname, type, _relaxed) \ + +GEN_XCHG_HELPERS(i8, s8) +GEN_XCHG_HELPERS(i16, s16) +GEN_XCHG_HELPERS(ptr, const void *) + +/* + * try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding try_cmpxchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +#define GEN_TRY_CMPXCHG_HELPER(tname, type, suffix) \ +__rust_helper bool \ +rust_helper_atomic_##tname##_try_cmpxchg##suffix(type *ptr, type *old, type new)\ +{ \ + return try_cmpxchg##suffix(ptr, old, new); \ +} + +#define GEN_TRY_CMPXCHG_HELPERS(tname, type) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, ) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _acquire) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _release) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _relaxed) \ + +GEN_TRY_CMPXCHG_HELPERS(i8, s8) +GEN_TRY_CMPXCHG_HELPERS(i16, s16) +GEN_TRY_CMPXCHG_HELPERS(ptr, const void *) diff --git a/rust/helpers/auxiliary.c b/rust/helpers/auxiliary.c new file mode 100644 index 000000000..dd1c13084 --- /dev/null +++ b/rust/helpers/auxiliary.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void +rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev) +{ + return auxiliary_device_uninit(adev); +} + +__rust_helper void +rust_helper_auxiliary_device_delete(struct auxiliary_device *adev) +{ + return auxiliary_device_delete(adev); +} diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c new file mode 100644 index 000000000..dbc7a3017 --- /dev/null +++ b/rust/helpers/barrier.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + +__rust_helper void rust_helper_smp_mb(void) +{ + smp_mb(); +} + +__rust_helper void rust_helper_smp_wmb(void) +{ + smp_wmb(); +} + +__rust_helper void rust_helper_smp_rmb(void) +{ + smp_rmb(); +} diff --git a/rust/helpers/binder.c b/rust/helpers/binder.c new file mode 100644 index 000000000..a2327f1b3 --- /dev/null +++ b/rust/helpers/binder.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2025 Google LLC. + */ + +#include +#include + +__rust_helper unsigned long rust_helper_list_lru_count(struct list_lru *lru) +{ + return list_lru_count(lru); +} + +__rust_helper unsigned long rust_helper_list_lru_walk(struct list_lru *lru, + list_lru_walk_cb isolate, + void *cb_arg, + unsigned long nr_to_walk) +{ + return list_lru_walk(lru, isolate, cb_arg, nr_to_walk); +} + +__rust_helper void rust_helper_init_task_work(struct callback_head *twork, + task_work_func_t func) +{ + init_task_work(twork, func); +} diff --git a/rust/helpers/bitmap.c b/rust/helpers/bitmap.c new file mode 100644 index 000000000..e4e9f4361 --- /dev/null +++ b/rust/helpers/bitmap.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_bitmap_copy_and_extend(unsigned long *to, const unsigned long *from, + unsigned int count, unsigned int size) +{ + bitmap_copy_and_extend(to, from, count, size); +} diff --git a/rust/helpers/bitops.c b/rust/helpers/bitops.c new file mode 100644 index 000000000..271b8a712 --- /dev/null +++ b/rust/helpers/bitops.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper +void rust_helper___set_bit(unsigned long nr, unsigned long *addr) +{ + __set_bit(nr, addr); +} + +__rust_helper +void rust_helper___clear_bit(unsigned long nr, unsigned long *addr) +{ + __clear_bit(nr, addr); +} + +__rust_helper +void rust_helper_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + set_bit(nr, addr); +} + +__rust_helper +void rust_helper_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + clear_bit(nr, addr); +} + +/* + * The rust_helper_ prefix is intentionally omitted below so that the + * declarations in include/linux/find.h are compatible with these helpers. + * + * Note that the below #ifdefs mean that the helper is only created if C does + * not provide a definition. + */ +#ifdef find_first_zero_bit +__rust_helper +unsigned long _find_first_zero_bit(const unsigned long *p, unsigned long size) +{ + return find_first_zero_bit(p, size); +} +#endif /* find_first_zero_bit */ + +#ifdef find_next_zero_bit +__rust_helper +unsigned long _find_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + return find_next_zero_bit(addr, size, offset); +} +#endif /* find_next_zero_bit */ + +#ifdef find_first_bit +__rust_helper +unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) +{ + return find_first_bit(addr, size); +} +#endif /* find_first_bit */ + +#ifdef find_next_bit +__rust_helper +unsigned long _find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + return find_next_bit(addr, size, offset); +} +#endif /* find_next_bit */ diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c new file mode 100644 index 000000000..20c512e46 --- /dev/null +++ b/rust/helpers/blk.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +{ + return blk_mq_rq_to_pdu(rq); +} + +__rust_helper struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) +{ + return blk_mq_rq_from_pdu(pdu); +} diff --git a/rust/helpers/bug.c b/rust/helpers/bug.c new file mode 100644 index 000000000..b51e60772 --- /dev/null +++ b/rust/helpers/bug.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper __noreturn void rust_helper_BUG(void) +{ + BUG(); +} + +__rust_helper bool rust_helper_WARN_ON(bool cond) +{ + return WARN_ON(cond); +} diff --git a/rust/helpers/build_assert.c b/rust/helpers/build_assert.c new file mode 100644 index 000000000..6a54b2680 --- /dev/null +++ b/rust/helpers/build_assert.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can + * use it in contexts where Rust expects a `usize` like slice (array) indices. + * `usize` is defined to be the same as C's `uintptr_t` type (can hold any + * pointer) but not necessarily the same as `size_t` (can hold the size of any + * single object). Most modern platforms use the same concrete integer type for + * both of them, but in case we find ourselves on a platform where + * that's not true, fail early instead of risking ABI or + * integer-overflow issues. + * + * If your platform fails this assertion, it means that you are in + * danger of integer-overflow bugs (even if you attempt to add + * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on + * your platform such that `size_t` matches `uintptr_t` (i.e., to increase + * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). + */ +static_assert( + sizeof(size_t) == sizeof(uintptr_t) && + __alignof__(size_t) == __alignof__(uintptr_t), + "Rust code expects C `size_t` to match Rust `usize`" +); diff --git a/rust/helpers/build_bug.c b/rust/helpers/build_bug.c new file mode 100644 index 000000000..14dbc55bb --- /dev/null +++ b/rust/helpers/build_bug.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper const char *rust_helper_errname(int err) +{ + return errname(err); +} diff --git a/rust/helpers/clk.c b/rust/helpers/clk.c new file mode 100644 index 000000000..15fd7e469 --- /dev/null +++ b/rust/helpers/clk.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * The "inline" implementation of below helpers are only available when + * CONFIG_HAVE_CLK or CONFIG_HAVE_CLK_PREPARE aren't set. + */ +#ifndef CONFIG_HAVE_CLK +__rust_helper struct clk *rust_helper_clk_get(struct device *dev, + const char *id) +{ + return clk_get(dev, id); +} + +__rust_helper void rust_helper_clk_put(struct clk *clk) +{ + clk_put(clk); +} + +__rust_helper int rust_helper_clk_enable(struct clk *clk) +{ + return clk_enable(clk); +} + +__rust_helper void rust_helper_clk_disable(struct clk *clk) +{ + clk_disable(clk); +} + +__rust_helper unsigned long rust_helper_clk_get_rate(struct clk *clk) +{ + return clk_get_rate(clk); +} + +__rust_helper int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate) +{ + return clk_set_rate(clk, rate); +} +#endif + +#ifndef CONFIG_HAVE_CLK_PREPARE +__rust_helper int rust_helper_clk_prepare(struct clk *clk) +{ + return clk_prepare(clk); +} + +__rust_helper void rust_helper_clk_unprepare(struct clk *clk) +{ + clk_unprepare(clk); +} +#endif + +__rust_helper struct clk *rust_helper_clk_get_optional(struct device *dev, + const char *id) +{ + return clk_get_optional(dev, id); +} + +__rust_helper int rust_helper_clk_prepare_enable(struct clk *clk) +{ + return clk_prepare_enable(clk); +} + +__rust_helper void rust_helper_clk_disable_unprepare(struct clk *clk) +{ + clk_disable_unprepare(clk); +} diff --git a/rust/helpers/completion.c b/rust/helpers/completion.c new file mode 100644 index 000000000..0126767cc --- /dev/null +++ b/rust/helpers/completion.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_completion(struct completion *x) +{ + init_completion(x); +} diff --git a/rust/helpers/cpu.c b/rust/helpers/cpu.c new file mode 100644 index 000000000..5759349b2 --- /dev/null +++ b/rust/helpers/cpu.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper unsigned int rust_helper_raw_smp_processor_id(void) +{ + return raw_smp_processor_id(); +} diff --git a/rust/helpers/cpufreq.c b/rust/helpers/cpufreq.c new file mode 100644 index 000000000..0e16aeef2 --- /dev/null +++ b/rust/helpers/cpufreq.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_CPU_FREQ +__rust_helper void +rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy) +{ + cpufreq_register_em_with_opp(policy); +} +#endif diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c new file mode 100644 index 000000000..5deced5b9 --- /dev/null +++ b/rust/helpers/cpumask.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + cpumask_set_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper___cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + __cpumask_set_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + cpumask_clear_cpu(cpu, dstp); +} + +__rust_helper +void rust_helper___cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + __cpumask_clear_cpu(cpu, dstp); +} + +__rust_helper +bool rust_helper_cpumask_test_cpu(int cpu, struct cpumask *srcp) +{ + return cpumask_test_cpu(cpu, srcp); +} + +__rust_helper +void rust_helper_cpumask_setall(struct cpumask *dstp) +{ + cpumask_setall(dstp); +} + +__rust_helper +bool rust_helper_cpumask_empty(struct cpumask *srcp) +{ + return cpumask_empty(srcp); +} + +__rust_helper +bool rust_helper_cpumask_full(struct cpumask *srcp) +{ + return cpumask_full(srcp); +} + +__rust_helper +unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) +{ + return cpumask_weight(srcp); +} + +__rust_helper +void rust_helper_cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) +{ + cpumask_copy(dstp, srcp); +} + +__rust_helper +bool rust_helper_alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var(mask, flags); +} + +__rust_helper +bool rust_helper_zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return zalloc_cpumask_var(mask, flags); +} + +#ifndef CONFIG_CPUMASK_OFFSTACK +__rust_helper +void rust_helper_free_cpumask_var(cpumask_var_t mask) +{ + free_cpumask_var(mask); +} +#endif diff --git a/rust/helpers/cred.c b/rust/helpers/cred.c new file mode 100644 index 000000000..a56a7b753 --- /dev/null +++ b/rust/helpers/cred.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper const struct cred *rust_helper_get_cred(const struct cred *cred) +{ + return get_cred(cred); +} + +__rust_helper void rust_helper_put_cred(const struct cred *cred) +{ + put_cred(cred); +} diff --git a/rust/helpers/device.c b/rust/helpers/device.c new file mode 100644 index 000000000..3be4ee590 --- /dev/null +++ b/rust/helpers/device.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_devm_add_action(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action(dev, action, data); +} + +__rust_helper int rust_helper_devm_add_action_or_reset(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action_or_reset(dev, action, data); +} + +__rust_helper void *rust_helper_dev_get_drvdata(const struct device *dev) +{ + return dev_get_drvdata(dev); +} + +__rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data) +{ + dev_set_drvdata(dev, data); +} + +__rust_helper const char *rust_helper_dev_name(const struct device *dev) +{ + return dev_name(dev); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 000000000..71914d824 --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +__rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c new file mode 100644 index 000000000..9fbeb507b --- /dev/null +++ b/rust/helpers/dma.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, + gfp_t flag, unsigned long attrs) +{ + return dma_alloc_attrs(dev, size, dma_handle, flag, attrs); +} + +__rust_helper void rust_helper_dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, + dma_addr_t dma_handle, + unsigned long attrs) +{ + dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs); +} + +__rust_helper int rust_helper_dma_set_mask_and_coherent(struct device *dev, + u64 mask) +{ + return dma_set_mask_and_coherent(dev, mask); +} + +__rust_helper int rust_helper_dma_set_mask(struct device *dev, u64 mask) +{ + return dma_set_mask(dev, mask); +} + +__rust_helper int rust_helper_dma_set_coherent_mask(struct device *dev, u64 mask) +{ + return dma_set_coherent_mask(dev, mask); +} + +__rust_helper int rust_helper_dma_map_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, unsigned long attrs) +{ + return dma_map_sgtable(dev, sgt, dir, attrs); +} + +__rust_helper size_t rust_helper_dma_max_mapping_size(struct device *dev) +{ + return dma_max_mapping_size(dev); +} + +__rust_helper void rust_helper_dma_set_max_seg_size(struct device *dev, + unsigned int size) +{ + dma_set_max_seg_size(dev, size); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c new file mode 100644 index 000000000..65f3f22b0 --- /dev/null +++ b/rust/helpers/drm.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#ifdef CONFIG_DRM + +__rust_helper void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) +{ + drm_gem_object_get(obj); +} + +__rust_helper void rust_helper_drm_gem_object_put(struct drm_gem_object *obj) +{ + drm_gem_object_put(obj); +} + +__rust_helper __u64 +rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) +{ + return drm_vma_node_offset_addr(node); +} + +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +__rust_helper void +rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +__rust_helper struct sg_table * +rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 000000000..4130b6325 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +#ifdef CONFIG_RUST_DRM_GPUVM + +#include + +__rust_helper +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +__rust_helper +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +__rust_helper +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif // CONFIG_RUST_DRM_GPUVM diff --git a/rust/helpers/err.c b/rust/helpers/err.c new file mode 100644 index 000000000..2872158e3 --- /dev/null +++ b/rust/helpers/err.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper __force void *rust_helper_ERR_PTR(long err) +{ + return ERR_PTR(err); +} + +__rust_helper bool rust_helper_IS_ERR(__force const void *ptr) +{ + return IS_ERR(ptr); +} + +__rust_helper long rust_helper_PTR_ERR(__force const void *ptr) +{ + return PTR_ERR(ptr); +} diff --git a/rust/helpers/fs.c b/rust/helpers/fs.c new file mode 100644 index 000000000..789d60fb8 --- /dev/null +++ b/rust/helpers/fs.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2024 Google LLC. + */ + +#include + +__rust_helper struct file *rust_helper_get_file(struct file *f) +{ + return get_file(f); +} diff --git a/rust/helpers/fwctl.c b/rust/helpers/fwctl.c new file mode 100644 index 000000000..c7eecd433 --- /dev/null +++ b/rust/helpers/fwctl.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#if IS_ENABLED(CONFIG_RUST_FWCTL_ABSTRACTIONS) + +__rust_helper struct fwctl_device *rust_helper_fwctl_get(struct fwctl_device *fwctl) +{ + return fwctl_get(fwctl); +} + +__rust_helper void rust_helper_fwctl_put(struct fwctl_device *fwctl) +{ + fwctl_put(fwctl); +} + +#endif diff --git a/rust/helpers/gpu.c b/rust/helpers/gpu.c new file mode 100644 index 000000000..a25448d54 --- /dev/null +++ b/rust/helpers/gpu.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_GPU_BUDDY + +__rust_helper u64 rust_helper_gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return gpu_buddy_block_offset(block); +} + +__rust_helper unsigned int rust_helper_gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_order(block); +} + +#endif /* CONFIG_GPU_BUDDY */ diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c new file mode 100644 index 000000000..440fb7638 --- /dev/null +++ b/rust/helpers/helpers.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions + * cannot be called either. This file explicitly creates functions ("helpers") + * that wrap those so that they can be called from Rust. + * + * Sorted alphabetically. + */ + +#include + +#ifdef __BINDGEN__ +// Omit `inline` for bindgen as it ignores inline functions. +#define __rust_helper +#else +// The helper functions are all inline functions. +// +// We use `__always_inline` here to bypass LLVM inlining checks, in case the +// helpers are inlined directly into Rust CGUs. +// +// The LLVM inlining checks are false positives: +// * LLVM doesn't want to inline functions compiled with +// `-fno-delete-null-pointer-checks` with code compiled without. +// The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay +// since this is one of the hardening features that does not change the ABI, +// and we shouldn't have null pointer dereferences in these helpers. +// * LLVM doesn't want to inline functions with different list of builtins. C +// side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they +// should be compatible, but LLVM does not perform inlining due to attributes +// mismatch. +// * clang and Rust doesn't have the exact target string. Clang generates +// `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will +// complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always +// enable these features, so they are in fact the same target string, but +// LLVM doesn't understand this and so inlining is inhibited. This can be +// bypassed with `--ignore-tti-inline-compatible`, but this is a hidden +// option. +#define __rust_helper __always_inline +#endif + +#include "atomic.c" +#include "atomic_ext.c" +#include "auxiliary.c" +#include "barrier.c" +#include "binder.c" +#include "bitmap.c" +#include "bitops.c" +#include "blk.c" +#include "bug.c" +#include "build_assert.c" +#include "build_bug.c" +#include "clk.c" +#include "completion.c" +#include "cpu.c" +#include "cpufreq.c" +#include "cpumask.c" +#include "cred.c" +#include "device.c" +#include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER +#include "dma-resv.c" +#endif +#include "drm.c" +#include "drm_gpuvm.c" +#include "err.c" +#include "fs.c" +#include "fwctl.c" +#include "gpu.c" +#include "interrupt.c" +#include "io.c" +#include "irq.c" +#include "jump_label.c" +#include "kunit.c" +#include "list.c" +#include "maple_tree.c" +#include "mm.c" +#include "mutex.c" +#include "net/genetlink.c" +#include "of.c" +#include "page.c" +#include "pci.c" +#include "pid_namespace.c" +#include "platform.c" +#include "poll.c" +#include "processor.c" +#include "property.c" +#include "pwm.c" +#include "rbtree.c" +#include "rcu.c" +#include "refcount.c" +#include "regulator.c" +#include "scatterlist.c" +#include "security.c" +#include "serdev.c" +#include "signal.c" +#include "slab.c" +#include "spinlock.c" +#include "string.c" +#include "srcu.c" +#include "sync.c" +#include "task.c" +#include "time.c" +#include "uaccess.c" +#include "usb.c" +#include "vmalloc.c" +#include "wait.c" +#include "workqueue.c" +#include "xarray.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000..695954986 --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} diff --git a/rust/helpers/io.c b/rust/helpers/io.c new file mode 100644 index 000000000..308950aae --- /dev/null +++ b/rust/helpers/io.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#ifdef CONFIG_HAS_IOMEM +__rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) +{ + return ioremap(offset, size); +} + +__rust_helper void __iomem *rust_helper_ioremap_np(phys_addr_t offset, + size_t size) +{ + return ioremap_np(offset, size); +} + +__rust_helper void rust_helper_iounmap(void __iomem *addr) +{ + iounmap(addr); +} +#endif /* CONFIG_HAS_IOMEM */ + +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} + +__rust_helper u8 rust_helper_readb(const void __iomem *addr) +{ + return readb(addr); +} + +__rust_helper u16 rust_helper_readw(const void __iomem *addr) +{ + return readw(addr); +} + +__rust_helper u32 rust_helper_readl(const void __iomem *addr) +{ + return readl(addr); +} + +#ifdef CONFIG_64BIT +__rust_helper u64 rust_helper_readq(const void __iomem *addr) +{ + return readq(addr); +} +#endif + +__rust_helper void rust_helper_writeb(u8 value, void __iomem *addr) +{ + writeb(value, addr); +} + +__rust_helper void rust_helper_writew(u16 value, void __iomem *addr) +{ + writew(value, addr); +} + +__rust_helper void rust_helper_writel(u32 value, void __iomem *addr) +{ + writel(value, addr); +} + +#ifdef CONFIG_64BIT +__rust_helper void rust_helper_writeq(u64 value, void __iomem *addr) +{ + writeq(value, addr); +} +#endif + +__rust_helper u8 rust_helper_readb_relaxed(const void __iomem *addr) +{ + return readb_relaxed(addr); +} + +__rust_helper u16 rust_helper_readw_relaxed(const void __iomem *addr) +{ + return readw_relaxed(addr); +} + +__rust_helper u32 rust_helper_readl_relaxed(const void __iomem *addr) +{ + return readl_relaxed(addr); +} + +#ifdef CONFIG_64BIT +__rust_helper u64 rust_helper_readq_relaxed(const void __iomem *addr) +{ + return readq_relaxed(addr); +} +#endif + +__rust_helper void rust_helper_writeb_relaxed(u8 value, void __iomem *addr) +{ + writeb_relaxed(value, addr); +} + +__rust_helper void rust_helper_writew_relaxed(u16 value, void __iomem *addr) +{ + writew_relaxed(value, addr); +} + +__rust_helper void rust_helper_writel_relaxed(u32 value, void __iomem *addr) +{ + writel_relaxed(value, addr); +} + +#ifdef CONFIG_64BIT +__rust_helper void rust_helper_writeq_relaxed(u64 value, void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +#endif + +__rust_helper resource_size_t rust_helper_resource_size(struct resource *res) +{ + return resource_size(res); +} + +__rust_helper struct resource * +rust_helper_request_mem_region(resource_size_t start, resource_size_t n, + const char *name) +{ + return request_mem_region(start, n, name); +} + +__rust_helper void rust_helper_release_mem_region(resource_size_t start, + resource_size_t n) +{ + release_mem_region(start, n); +} + +__rust_helper struct resource *rust_helper_request_region(resource_size_t start, + resource_size_t n, + const char *name) +{ + return request_region(start, n, name); +} + +__rust_helper struct resource * +rust_helper_request_muxed_region(resource_size_t start, resource_size_t n, + const char *name) +{ + return request_muxed_region(start, n, name); +} + +__rust_helper void rust_helper_release_region(resource_size_t start, + resource_size_t n) +{ + release_region(start, n); +} diff --git a/rust/helpers/irq.c b/rust/helpers/irq.c new file mode 100644 index 000000000..a61fad333 --- /dev/null +++ b/rust/helpers/irq.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_request_irq(unsigned int irq, + irq_handler_t handler, + unsigned long flags, const char *name, + void *dev) +{ + return request_irq(irq, handler, flags, name, dev); +} diff --git a/rust/helpers/jump_label.c b/rust/helpers/jump_label.c new file mode 100644 index 000000000..7ca384e73 --- /dev/null +++ b/rust/helpers/jump_label.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2024 Google LLC. + */ + +#include + +#ifndef CONFIG_JUMP_LABEL +__rust_helper int rust_helper_static_key_count(struct static_key *key) +{ + return static_key_count(key); +} +#endif diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c new file mode 100644 index 000000000..cafb94b67 --- /dev/null +++ b/rust/helpers/kunit.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper struct kunit *rust_helper_kunit_get_current_test(void) +{ + return kunit_get_current_test(); +} diff --git a/rust/helpers/list.c b/rust/helpers/list.c new file mode 100644 index 000000000..18095a559 --- /dev/null +++ b/rust/helpers/list.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Helpers for C circular doubly linked list implementation. + */ + +#include + +__rust_helper void rust_helper_INIT_LIST_HEAD(struct list_head *list) +{ + INIT_LIST_HEAD(list); +} + +__rust_helper void rust_helper_list_add_tail(struct list_head *new, struct list_head *head) +{ + list_add_tail(new, head); +} diff --git a/rust/helpers/maple_tree.c b/rust/helpers/maple_tree.c new file mode 100644 index 000000000..5586486a7 --- /dev/null +++ b/rust/helpers/maple_tree.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mt_init_flags(struct maple_tree *mt, + unsigned int flags) +{ + mt_init_flags(mt, flags); +} diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c new file mode 100644 index 000000000..b5540997b --- /dev/null +++ b/rust/helpers/mm.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_mmgrab(struct mm_struct *mm) +{ + mmgrab(mm); +} + +__rust_helper void rust_helper_mmdrop(struct mm_struct *mm) +{ + mmdrop(mm); +} + +__rust_helper void rust_helper_mmget(struct mm_struct *mm) +{ + mmget(mm); +} + +__rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) +{ + return mmget_not_zero(mm); +} + +__rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) +{ + mmap_read_lock(mm); +} + +__rust_helper bool rust_helper_mmap_read_trylock(struct mm_struct *mm) +{ + return mmap_read_trylock(mm); +} + +__rust_helper void rust_helper_mmap_read_unlock(struct mm_struct *mm) +{ + mmap_read_unlock(mm); +} + +__rust_helper struct vm_area_struct * +rust_helper_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + return vma_lookup(mm, addr); +} + +__rust_helper void rust_helper_vma_end_read(struct vm_area_struct *vma) +{ + vma_end_read(vma); +} diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c new file mode 100644 index 000000000..1b07d6e64 --- /dev/null +++ b/rust/helpers/mutex.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_mutex_lock(struct mutex *lock) +{ + mutex_lock(lock); +} + +__rust_helper int rust_helper_mutex_trylock(struct mutex *lock) +{ + return mutex_trylock(lock); +} + +__rust_helper void rust_helper___mutex_init(struct mutex *mutex, + const char *name, + struct lock_class_key *key) +{ + __mutex_init(mutex, name, key); +} + +__rust_helper void rust_helper_mutex_assert_is_held(struct mutex *mutex) +{ + lockdep_assert_held(mutex); +} + +__rust_helper void rust_helper_mutex_destroy(struct mutex *lock) +{ + mutex_destroy(lock); +} diff --git a/rust/helpers/net/genetlink.c b/rust/helpers/net/genetlink.c new file mode 100644 index 000000000..3530b69f6 --- /dev/null +++ b/rust/helpers/net/genetlink.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2026 Google LLC. + */ + +#include + +#ifdef CONFIG_NET + +__rust_helper struct sk_buff *rust_helper_genlmsg_new(size_t payload, gfp_t flags) +{ + return genlmsg_new(payload, flags); +} + +__rust_helper +int rust_helper_genlmsg_multicast(const struct genl_family *family, + struct sk_buff *skb, u32 portid, + unsigned int group, gfp_t flags) +{ + return genlmsg_multicast(family, skb, portid, group, flags); +} + +__rust_helper void rust_helper_genlmsg_cancel(struct sk_buff *skb, void *hdr) +{ + genlmsg_cancel(skb, hdr); +} + +__rust_helper void rust_helper_genlmsg_end(struct sk_buff *skb, void *hdr) +{ + genlmsg_end(skb, hdr); +} + +__rust_helper void rust_helper_nlmsg_free(struct sk_buff *skb) +{ + nlmsg_free(skb); +} + +__rust_helper +int rust_helper_genl_has_listeners(const struct genl_family *family, + struct net *net, unsigned int group) +{ + return genl_has_listeners(family, net, group); +} + +#endif diff --git a/rust/helpers/of.c b/rust/helpers/of.c new file mode 100644 index 000000000..8f62ca69e --- /dev/null +++ b/rust/helpers/of.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) +{ + return is_of_node(fwnode); +} diff --git a/rust/helpers/page.c b/rust/helpers/page.c new file mode 100644 index 000000000..f8463fbed --- /dev/null +++ b/rust/helpers/page.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +__rust_helper struct page *rust_helper_alloc_pages(gfp_t gfp_mask, + unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} + +__rust_helper void *rust_helper_kmap_local_page(struct page *page) +{ + return kmap_local_page(page); +} + +__rust_helper void rust_helper_kunmap_local(const void *addr) +{ + kunmap_local(addr); +} + +#ifndef NODE_NOT_IN_PAGE_FLAGS +__rust_helper int rust_helper_page_to_nid(const struct page *page) +{ + return page_to_nid(page); +} +#endif diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c new file mode 100644 index 000000000..a714cc2bf --- /dev/null +++ b/rust/helpers/pci.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper u16 rust_helper_pci_dev_id(struct pci_dev *dev) +{ + return PCI_DEVID(dev->bus->number, dev->devfn); +} + +__rust_helper resource_size_t +rust_helper_pci_resource_start(struct pci_dev *pdev, int bar) +{ + return pci_resource_start(pdev, bar); +} + +__rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, + int bar) +{ + return pci_resource_len(pdev, bar); +} + +__rust_helper bool rust_helper_dev_is_pci(const struct device *dev) +{ + return dev_is_pci(dev); +} + +__rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev) +{ + return pci_irq_type(pdev); +} + +#ifndef CONFIG_PCI_IOV +__rust_helper unsigned int +rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev) +{ + return pci_sriov_get_totalvfs(pdev); +} +#endif + +#ifndef CONFIG_PCI_MSI +__rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, + unsigned int min_vecs, + unsigned int max_vecs, + unsigned int flags) +{ + return pci_alloc_irq_vectors(dev, min_vecs, max_vecs, flags); +} + +__rust_helper void rust_helper_pci_free_irq_vectors(struct pci_dev *dev) +{ + pci_free_irq_vectors(dev); +} + +__rust_helper int rust_helper_pci_irq_vector(struct pci_dev *pdev, + unsigned int nvec) +{ + return pci_irq_vector(pdev, nvec); +} +#endif diff --git a/rust/helpers/pid_namespace.c b/rust/helpers/pid_namespace.c new file mode 100644 index 000000000..f46ab779b --- /dev/null +++ b/rust/helpers/pid_namespace.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper struct pid_namespace * +rust_helper_get_pid_ns(struct pid_namespace *ns) +{ + return get_pid_ns(ns); +} + +__rust_helper void rust_helper_put_pid_ns(struct pid_namespace *ns) +{ + put_pid_ns(ns); +} + +/* Get a reference on a task's pid namespace. */ +__rust_helper struct pid_namespace * +rust_helper_task_get_pid_ns(struct task_struct *task) +{ + struct pid_namespace *pid_ns; + + guard(rcu)(); + pid_ns = task_active_pid_ns(task); + if (pid_ns) + get_pid_ns(pid_ns); + return pid_ns; +} diff --git a/rust/helpers/platform.c b/rust/helpers/platform.c new file mode 100644 index 000000000..188b3240f --- /dev/null +++ b/rust/helpers/platform.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_dev_is_platform(const struct device *dev) +{ + return dev_is_platform(dev); +} diff --git a/rust/helpers/poll.c b/rust/helpers/poll.c new file mode 100644 index 000000000..78b3839b5 --- /dev/null +++ b/rust/helpers/poll.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_poll_wait(struct file *filp, + wait_queue_head_t *wait_address, + poll_table *p) +{ + poll_wait(filp, wait_address, p); +} diff --git a/rust/helpers/processor.c b/rust/helpers/processor.c new file mode 100644 index 000000000..76fadbb64 --- /dev/null +++ b/rust/helpers/processor.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_cpu_relax(void) +{ + cpu_relax(); +} diff --git a/rust/helpers/property.c b/rust/helpers/property.c new file mode 100644 index 000000000..8fb990053 --- /dev/null +++ b/rust/helpers/property.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode) +{ + fwnode_handle_put(fwnode); +} diff --git a/rust/helpers/pwm.c b/rust/helpers/pwm.c new file mode 100644 index 000000000..eb24d2ea8 --- /dev/null +++ b/rust/helpers/pwm.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2025 Samsung Electronics Co., Ltd. +// Author: Michal Wilczynski + +#include + +__rust_helper struct device *rust_helper_pwmchip_parent(const struct pwm_chip *chip) +{ + return pwmchip_parent(chip); +} + +__rust_helper void *rust_helper_pwmchip_get_drvdata(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} + +__rust_helper void rust_helper_pwmchip_set_drvdata(struct pwm_chip *chip, void *data) +{ + pwmchip_set_drvdata(chip, data); +} diff --git a/rust/helpers/rbtree.c b/rust/helpers/rbtree.c new file mode 100644 index 000000000..a85defb22 --- /dev/null +++ b/rust/helpers/rbtree.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_rb_link_node(struct rb_node *node, + struct rb_node *parent, + struct rb_node **rb_link) +{ + rb_link_node(node, parent, rb_link); +} + +__rust_helper struct rb_node *rust_helper_rb_first(const struct rb_root *root) +{ + return rb_first(root); +} + +__rust_helper struct rb_node *rust_helper_rb_last(const struct rb_root *root) +{ + return rb_last(root); +} diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c new file mode 100644 index 000000000..481274c05 --- /dev/null +++ b/rust/helpers/rcu.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_rcu_read_lock(void) +{ + rcu_read_lock(); +} + +__rust_helper void rust_helper_rcu_read_unlock(void) +{ + rcu_read_unlock(); +} diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c new file mode 100644 index 000000000..36334a674 --- /dev/null +++ b/rust/helpers/refcount.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper refcount_t rust_helper_REFCOUNT_INIT(int n) +{ + return (refcount_t)REFCOUNT_INIT(n); +} + +__rust_helper void rust_helper_refcount_set(refcount_t *r, int n) +{ + refcount_set(r, n); +} + +__rust_helper void rust_helper_refcount_inc(refcount_t *r) +{ + refcount_inc(r); +} + +__rust_helper void rust_helper_refcount_dec(refcount_t *r) +{ + refcount_dec(r); +} + +__rust_helper bool rust_helper_refcount_dec_and_test(refcount_t *r) +{ + return refcount_dec_and_test(r); +} diff --git a/rust/helpers/regulator.c b/rust/helpers/regulator.c new file mode 100644 index 000000000..9ec5237f4 --- /dev/null +++ b/rust/helpers/regulator.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifndef CONFIG_REGULATOR + +__rust_helper void rust_helper_regulator_put(struct regulator *regulator) +{ + regulator_put(regulator); +} + +__rust_helper int rust_helper_regulator_set_voltage(struct regulator *regulator, + int min_uV, int max_uV) +{ + return regulator_set_voltage(regulator, min_uV, max_uV); +} + +__rust_helper int rust_helper_regulator_get_voltage(struct regulator *regulator) +{ + return regulator_get_voltage(regulator); +} + +__rust_helper struct regulator *rust_helper_regulator_get(struct device *dev, + const char *id) +{ + return regulator_get(dev, id); +} + +__rust_helper int rust_helper_regulator_enable(struct regulator *regulator) +{ + return regulator_enable(regulator); +} + +__rust_helper int rust_helper_regulator_disable(struct regulator *regulator) +{ + return regulator_disable(regulator); +} + +__rust_helper int rust_helper_regulator_is_enabled(struct regulator *regulator) +{ + return regulator_is_enabled(regulator); +} + +__rust_helper int rust_helper_devm_regulator_get_enable(struct device *dev, + const char *id) +{ + return devm_regulator_get_enable(dev, id); +} + +__rust_helper int +rust_helper_devm_regulator_get_enable_optional(struct device *dev, + const char *id) +{ + return devm_regulator_get_enable_optional(dev, id); +} + +#endif diff --git a/rust/helpers/scatterlist.c b/rust/helpers/scatterlist.c new file mode 100644 index 000000000..f3c41ea5e --- /dev/null +++ b/rust/helpers/scatterlist.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper dma_addr_t rust_helper_sg_dma_address(struct scatterlist *sg) +{ + return sg_dma_address(sg); +} + +__rust_helper unsigned int rust_helper_sg_dma_len(struct scatterlist *sg) +{ + return sg_dma_len(sg); +} + +__rust_helper struct scatterlist *rust_helper_sg_next(struct scatterlist *sg) +{ + return sg_next(sg); +} + +__rust_helper void rust_helper_dma_unmap_sgtable(struct device *dev, + struct sg_table *sgt, + enum dma_data_direction dir, + unsigned long attrs) +{ + return dma_unmap_sgtable(dev, sgt, dir, attrs); +} diff --git a/rust/helpers/security.c b/rust/helpers/security.c new file mode 100644 index 000000000..8d0a25fcf --- /dev/null +++ b/rust/helpers/security.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifndef CONFIG_SECURITY +__rust_helper void rust_helper_security_cred_getsecid(const struct cred *c, + u32 *secid) +{ + security_cred_getsecid(c, secid); +} + +__rust_helper int rust_helper_security_secid_to_secctx(u32 secid, + struct lsm_context *cp) +{ + return security_secid_to_secctx(secid, cp); +} + +__rust_helper void rust_helper_security_release_secctx(struct lsm_context *cp) +{ + security_release_secctx(cp); +} + +__rust_helper int +rust_helper_security_binder_set_context_mgr(const struct cred *mgr) +{ + return security_binder_set_context_mgr(mgr); +} + +__rust_helper int +rust_helper_security_binder_transaction(const struct cred *from, + const struct cred *to) +{ + return security_binder_transaction(from, to); +} + +__rust_helper int +rust_helper_security_binder_transfer_binder(const struct cred *from, + const struct cred *to) +{ + return security_binder_transfer_binder(from, to); +} + +__rust_helper int rust_helper_security_binder_transfer_file( + const struct cred *from, const struct cred *to, const struct file *file) +{ + return security_binder_transfer_file(from, to, file); +} +#endif diff --git a/rust/helpers/serdev.c b/rust/helpers/serdev.c new file mode 100644 index 000000000..c52b78ca3 --- /dev/null +++ b/rust/helpers/serdev.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_serdev_device_driver_unregister(struct serdev_device_driver *sdrv) +{ + serdev_device_driver_unregister(sdrv); +} + +__rust_helper +void rust_helper_serdev_device_put(struct serdev_device *serdev) +{ + serdev_device_put(serdev); +} + +__rust_helper +void rust_helper_serdev_device_set_client_ops(struct serdev_device *serdev, + const struct serdev_device_ops *ops) +{ + serdev_device_set_client_ops(serdev, ops); +} diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c new file mode 100644 index 000000000..85111186c --- /dev/null +++ b/rust/helpers/signal.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_signal_pending(struct task_struct *t) +{ + return signal_pending(t); +} diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c new file mode 100644 index 000000000..9279f0824 --- /dev/null +++ b/rust/helpers/slab.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_krealloc_node_align(const void *objp, size_t new_size, unsigned long align, + gfp_t flags, int node) +{ + return krealloc_node_align(objp, new_size, align, flags, node); +} + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_kvrealloc_node_align(const void *p, size_t size, unsigned long align, + gfp_t flags, int node) +{ + return kvrealloc_node_align(p, size, align, flags, node); +} diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c new file mode 100644 index 000000000..d53400c15 --- /dev/null +++ b/rust/helpers/spinlock.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper___spin_lock_init(spinlock_t *lock, + const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_SPINLOCK +# if defined(CONFIG_PREEMPT_RT) + __spin_lock_init(lock, name, key, false); +# else /*!CONFIG_PREEMPT_RT */ + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +# endif /* CONFIG_PREEMPT_RT */ +#else /* !CONFIG_DEBUG_SPINLOCK */ + spin_lock_init(lock); +#endif /* CONFIG_DEBUG_SPINLOCK */ +} + +__rust_helper void rust_helper_spin_lock(spinlock_t *lock) +{ + spin_lock(lock); +} + +__rust_helper void rust_helper_spin_unlock(spinlock_t *lock) +{ + spin_unlock(lock); +} + +__rust_helper int rust_helper_spin_trylock(spinlock_t *lock) +{ + return spin_trylock(lock); +} + +__rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) +{ + lockdep_assert_held(lock); +} + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000..1a2f56364 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/helpers/string.c b/rust/helpers/string.c new file mode 100644 index 000000000..8ef30eb07 --- /dev/null +++ b/rust/helpers/string.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *rust_helper_memchr(const void *s, int c, size_t n) +{ + return memchr(s, c, n); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c new file mode 100644 index 000000000..4f474fe84 --- /dev/null +++ b/rust/helpers/sync.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_lockdep_register_key(struct lock_class_key *k) +{ + lockdep_register_key(k); +} + +__rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) +{ + lockdep_unregister_key(k); +} + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/helpers/task.c b/rust/helpers/task.c new file mode 100644 index 000000000..b46b1433a --- /dev/null +++ b/rust/helpers/task.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void rust_helper_might_resched(void) +{ + might_resched(); +} + +__rust_helper struct task_struct *rust_helper_get_current(void) +{ + return current; +} + +__rust_helper void rust_helper_get_task_struct(struct task_struct *t) +{ + get_task_struct(t); +} + +__rust_helper void rust_helper_put_task_struct(struct task_struct *t) +{ + put_task_struct(t); +} + +__rust_helper kuid_t rust_helper_task_uid(struct task_struct *task) +{ + return task_uid(task); +} + +#ifndef CONFIG_USER_NS +__rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) +{ + return from_kuid(to, uid); +} +#endif /* CONFIG_USER_NS */ + +__rust_helper bool rust_helper_uid_eq(kuid_t left, kuid_t right) +{ + return uid_eq(left, right); +} + +__rust_helper kuid_t rust_helper_current_euid(void) +{ + return current_euid(); +} + +__rust_helper struct user_namespace *rust_helper_current_user_ns(void) +{ + return current_user_ns(); +} + +__rust_helper pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk, + struct pid_namespace *ns) +{ + return task_tgid_nr_ns(tsk, ns); +} diff --git a/rust/helpers/time.c b/rust/helpers/time.c new file mode 100644 index 000000000..32f495970 --- /dev/null +++ b/rust/helpers/time.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +__rust_helper void rust_helper_fsleep(unsigned long usecs) +{ + fsleep(usecs); +} + +__rust_helper ktime_t rust_helper_ktime_get_real(void) +{ + return ktime_get_real(); +} + +__rust_helper ktime_t rust_helper_ktime_get_boottime(void) +{ + return ktime_get_boottime(); +} + +__rust_helper ktime_t rust_helper_ktime_get_clocktai(void) +{ + return ktime_get_clocktai(); +} + +__rust_helper s64 rust_helper_ktime_to_us(const ktime_t kt) +{ + return ktime_to_us(kt); +} + +__rust_helper s64 rust_helper_ktime_to_ms(const ktime_t kt) +{ + return ktime_to_ms(kt); +} + +__rust_helper void rust_helper_udelay(unsigned long usec) +{ + udelay(usec); +} diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c new file mode 100644 index 000000000..6e59cc9c6 --- /dev/null +++ b/rust/helpers/uaccess.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper unsigned long +rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return copy_from_user(to, from, n); +} + +__rust_helper unsigned long +rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return copy_to_user(to, from, n); +} + +#ifdef INLINE_COPY_USER +__rust_helper +unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return _inline_copy_from_user(to, from, n); +} + +__rust_helper +unsigned long rust_helper__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return _inline_copy_to_user(to, from, n); +} +#endif diff --git a/rust/helpers/usb.c b/rust/helpers/usb.c new file mode 100644 index 000000000..eff1cf7be --- /dev/null +++ b/rust/helpers/usb.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper struct usb_device * +rust_helper_interface_to_usbdev(struct usb_interface *intf) +{ + return interface_to_usbdev(intf); +} diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 000000000..6aed13292 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper void *__must_check __realloc_size(2) +rust_helper_vrealloc_node_align(const void *p, size_t size, unsigned long align, + gfp_t flags, int node) +{ + return vrealloc_node_align(p, size, align, flags, node); +} + +__rust_helper bool rust_helper_is_vmalloc_addr(const void *x) +{ + return is_vmalloc_addr(x); +} diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c new file mode 100644 index 000000000..2dde1e451 --- /dev/null +++ b/rust/helpers/wait.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +{ + init_wait(wq_entry); +} diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c new file mode 100644 index 000000000..ce1c3a5b2 --- /dev/null +++ b/rust/helpers/workqueue.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_init_work_with_key(struct work_struct *work, + work_func_t func, + bool onstack, + const char *name, + struct lock_class_key *key) +{ + __init_work(work, onstack); + work->data = (atomic_long_t)WORK_DATA_INIT(); + lockdep_init_map(&work->lockdep_map, name, key, 0); + INIT_LIST_HEAD(&work->entry); + work->func = func; +} diff --git a/rust/helpers/xarray.c b/rust/helpers/xarray.c new file mode 100644 index 000000000..08979b304 --- /dev/null +++ b/rust/helpers/xarray.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_xa_err(void *entry) +{ + return xa_err(entry); +} + +__rust_helper void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) +{ + return xa_init_flags(xa, flags); +} + +__rust_helper int rust_helper_xa_trylock(struct xarray *xa) +{ + return xa_trylock(xa); +} + +__rust_helper void rust_helper_xa_lock(struct xarray *xa) +{ + return xa_lock(xa); +} + +__rust_helper void rust_helper_xa_unlock(struct xarray *xa) +{ + return xa_unlock(xa); +} diff --git a/rust/kernel/.gitignore b/rust/kernel/.gitignore new file mode 100644 index 000000000..f636ad95a --- /dev/null +++ b/rust/kernel/.gitignore @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +/generated_arch_static_branch_asm.rs +/generated_arch_warn_asm.rs +/generated_arch_reachable_asm.rs diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test new file mode 100644 index 000000000..e6a5c7a79 --- /dev/null +++ b/rust/kernel/Kconfig.test @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig RUST_KUNIT_TESTS + bool "Rust KUnit tests" + depends on KUNIT && RUST + default KUNIT_ALL_TESTS + help + This menu collects all options for Rust KUnit tests. + See Documentation/rust/testing.rst for how to protect + unit tests with these options. + + Say Y here to enable Rust KUnit tests. + + If unsure, say N. + +if RUST_KUNIT_TESTS +config RUST_ALLOCATOR_KUNIT_TEST + bool "KUnit tests for Rust allocator API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust allocator API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KVEC_KUNIT_TEST + bool "KUnit tests for Rust KVec API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust KVec API. + These are only for development and testing, not for + regular kernel use cases. + + If unsure, say N. + +config RUST_BITMAP_KUNIT_TEST + bool "KUnit tests for Rust bitmap API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust bitmap API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KUNIT_SELFTEST + bool "KUnit selftests for Rust" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit selftests. These are only + for development and testing, not for regular kernel + use cases. + + If unsure, say N. + +config RUST_STR_KUNIT_TEST + bool "KUnit tests for Rust strings API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust strings API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_ATOMICS_KUNIT_TEST + bool "KUnit tests for Rust atomics API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust atomics API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_BITFIELD_KUNIT_TEST + bool "KUnit tests for the Rust `bitfield!` macro" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust `bitfield!` macro. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +endif diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs new file mode 100644 index 000000000..ea2ce61ee --- /dev/null +++ b/rust/kernel/acpi.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Advanced Configuration and Power Interface abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for ACPI drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// An ACPI device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::acpi_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `acpi_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::acpi_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); +} + +impl DeviceId { + const ACPI_ID_LEN: usize = 16; + + /// Create a new device id from an ACPI 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes"); + let mut acpi: bindings::acpi_device_id = pin_init::zeroed(); + let mut i = 0; + while i < src.len() { + acpi.id[i] = src[i]; + i += 1; + } + + Self(acpi) + } +} + +/// Create an ACPI `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! acpi_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("acpi", $crate::acpi::DeviceId, $($tt)*); + }; +} diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs new file mode 100644 index 000000000..21067bde6 --- /dev/null +++ b/rust/kernel/alloc.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of the kernel's memory allocation infrastructure. + +pub mod allocator; +pub mod kbox; +pub mod kvec; +pub mod layout; + +pub use self::kbox::Box; +pub use self::kbox::KBox; +pub use self::kbox::KVBox; +pub use self::kbox::VBox; + +pub use self::kvec::IntoIter; +pub use self::kvec::KVVec; +pub use self::kvec::KVec; +pub use self::kvec::VVec; +pub use self::kvec::Vec; + +/// Indicates an allocation error. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +use crate::prelude::*; + +use core::{ + alloc::Layout, + ptr::NonNull, // +}; + +/// Flags to be used when allocating memory. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`flags`] module. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(u32); + +impl Flags { + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> u32 { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Allocation flags. +/// +/// These are meant to be used in functions that can allocate memory. +pub mod flags { + use super::Flags; + + /// Zeroes out the allocated memory. + /// + /// This is normally or'd with other flags. + pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO); + + /// Allow the allocation to be in high memory. + /// + /// Allocations in high memory may not be mapped into the kernel's address space, so this can't + /// be used with `kmalloc` and other similar methods. + /// + /// This is normally or'd with other flags. + pub const __GFP_HIGHMEM: Flags = Flags(bindings::__GFP_HIGHMEM); + + /// Users can not sleep and need the allocation to succeed. + /// + /// A lower watermark is applied to allow access to "atomic reserves". The current + /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g. + /// `raw_spin_lock`). The same applies to [`GFP_NOWAIT`]. + pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC); + + /// Typical for kernel-internal allocations. The caller requires `ZONE_NORMAL` or a lower zone + /// for direct access but can direct reclaim. + pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL); + + /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg. + pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT); + + /// For kernel allocations that should not stall for direct reclaim, start physical IO or + /// use any filesystem callback. It is very likely to fail to allocate memory, even for very + /// small allocations. + pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); + + /// Suppresses allocation failure reports. + /// + /// This is normally or'd with other flags. + pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); +} + +/// Non Uniform Memory Access (NUMA) node identifier. +#[derive(Clone, Copy, PartialEq)] +pub struct NumaNode(i32); + +impl NumaNode { + /// Create a new NUMA node identifier (non-negative integer). + /// + /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is + /// specified. + pub fn new(node: i32) -> Result { + // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10. + if node < 0 || node >= bindings::MAX_NUMNODES as i32 { + return Err(EINVAL); + } + Ok(Self(node)) + } +} + +/// Specify necessary constant to pass the information to Allocator that the caller doesn't care +/// about the NUMA node to allocate memory from. +impl NumaNode { + /// No node preference. + pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE); +} + +/// The kernel's [`Allocator`] trait. +/// +/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described +/// via [`Layout`]. +/// +/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on +/// an object instance. +/// +/// In order to be able to support `#[derive(CoercePointee)]` later on, we need to avoid a design +/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind +/// of `self` parameter. +/// +/// # Safety +/// +/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed. +/// +/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`] +/// function of the same type. +/// +/// - Implementers must ensure that all trait functions abide by the guarantees documented in the +/// `# Guarantees` sections. +pub unsafe trait Allocator { + /// The minimum alignment satisfied by all allocations from this allocator. + /// + /// # Guarantees + /// + /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if + /// the requested layout has a smaller alignment. + const MIN_ALIGN: usize; + + /// Allocate memory based on `layout`, `flags` and `nid`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`] or [`Allocator::realloc`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// . + fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result, AllocError> { + // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a + // new memory allocation. + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) } + } + + /// Re-allocate an existing memory allocation to satisfy the requested `layout` and + /// a specific NUMA node request to allocate the memory for. + /// + /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of + /// hardware resources including processors, memory, and I/O buses, that comprise what is + /// commonly known as a NUMA node. + /// + /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer + /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care. + /// + /// If the requested size is zero, `realloc` behaves equivalent to `free`. + /// + /// If the requested size is larger than the size of the existing allocation, a successful call + /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but + /// may also be larger. + /// + /// If the requested size is smaller than the size of the existing allocation, `realloc` may or + /// may not shrink the buffer; this is implementation specific to the allocator. + /// + /// On allocation failure, the existing buffer, if any, remains valid. + /// + /// The buffer is represented as `NonNull<[u8]>`. + /// + /// # Safety + /// + /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation + /// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a + /// pointer returned by this [`Allocator`]. + /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and + /// `old_layout` is ignored. + /// - `old_layout` must match the `Layout` the allocation has been created with. + /// + /// # Guarantees + /// + /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then + /// it additionally guarantees that: + /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new + /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == + /// p[0..min(layout.size(), old_layout.size())]`. + /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError>; + + /// Free an existing memory allocation. + /// + /// # Safety + /// + /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`]; + /// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this + /// [`Allocator`]. + /// - `layout` must match the `Layout` the allocation has been created with. + /// - The memory allocation at `ptr` must never again be read from or written to. + unsafe fn free(ptr: NonNull, layout: Layout) { + // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this + // allocator. We are passing a `Layout` with the smallest possible alignment, so it is + // smaller than or equal to the alignment previously used with this allocation. + let _ = unsafe { + Self::realloc( + Some(ptr), + Layout::new::<()>(), + layout, + Flags(0), + NumaNode::NO_NODE, + ) + }; + } +} + +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs new file mode 100644 index 000000000..cd4203f27 --- /dev/null +++ b/rust/kernel/alloc/allocator.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + bindings, + page, // +}; + +use core::{ + alloc::Layout, + ptr::{ + self, + NonNull, // + }, // +}; + +const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; + +mod iter; +pub use self::iter::VmallocPageIter; + +/// The contiguous kernel allocator. +/// +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. +/// +/// For more details see [self]. +pub struct Kmalloc; + +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; + +/// # Invariants +/// +/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`. +struct ReallocFunc( + unsafe extern "C" fn( + *const crate::ffi::c_void, + usize, + crate::ffi::c_ulong, + u32, + crate::ffi::c_int, + ) -> *mut crate::ffi::c_void, +); + +impl ReallocFunc { + // INVARIANT: `krealloc_node_align` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc_node_align); + + // INVARIANT: `vrealloc_node_align` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc_node_align); + + // INVARIANT: `kvrealloc_node_align` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc_node_align); + + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + #[inline] + unsafe fn call( + &self, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + let size = layout.size(); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; + + // SAFETY: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } +} + +impl Kmalloc { + /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of + /// `layout`. + pub fn aligned_layout(layout: Layout) -> Layout { + // Note that `layout.size()` (after padding) is guaranteed to be a multiple of + // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return + // a properly aligned object (see comments in `kmalloc()` for more information). + layout.pad_to_align() + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + let layout = Kmalloc::aligned_layout(layout); + + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +impl Vmalloc { + /// Convert a pointer to a [`Vmalloc`] allocation to a [`page::BorrowedPage`]. + /// + /// # Examples + /// + /// ``` + /// # use core::ptr::{ + /// # from_mut, + /// # NonNull, // + /// # }; + /// # use kernel::page; + /// use kernel::alloc::allocator::Vmalloc; + /// + /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; + /// + /// { + /// // SAFETY: By the type invariant of `Box` the inner pointer of `vbox` is non-null. + /// let ptr = unsafe { NonNull::new_unchecked(from_mut(&mut *vbox)) }; + /// + /// // SAFETY: + /// // `ptr` is a valid pointer to a `Vmalloc` allocation. + /// // `ptr` is valid for the entire lifetime of `page`. + /// let page = unsafe { Vmalloc::to_page(ptr.cast()) }; + /// + /// // SAFETY: There is no concurrent read or write to the same page. + /// unsafe { page.fill_zero_raw(0, page::PAGE_SIZE)? }; + /// } + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a [`Vmalloc`] allocation. + /// - `ptr` must remain valid for the entire duration of `'a`. + pub unsafe fn to_page<'a>(ptr: NonNull) -> page::BorrowedPage<'a> { + // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. + let page = unsafe { bindings::vmalloc_to_page(ptr.as_ptr().cast()) }; + + // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid pointer + // to `Vmalloc` memory. + let page = unsafe { NonNull::new_unchecked(page) }; + + // SAFETY: + // - `page` is a valid pointer to a `struct page`, given that by the safety requirements of + // this function `ptr` is a valid pointer to a `Vmalloc` allocation. + // - By the safety requirements of this function `ptr` is valid for the entire lifetime of + // `'a`. + unsafe { page::BorrowedPage::from_raw(page) } + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + nid: NumaNode, + ) -> Result, AllocError> { + // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc` + // compatible layout. + let layout = Kmalloc::aligned_layout(layout); + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } + } +} + +#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)] +#[macros::kunit_tests(rust_allocator)] +mod tests { + use super::*; + use core::mem::MaybeUninit; + use kernel::prelude::*; + + #[test] + fn test_alignment() -> Result { + const TEST_SIZE: usize = 1024; + const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4; + + // These two structs are used to test allocating aligned memory. + // they don't need to be accessed, so they're marked as dead_code. + #[expect(dead_code)] + #[repr(align(128))] + struct Blob([u8; TEST_SIZE]); + #[expect(dead_code)] + #[repr(align(8192))] + struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]); + + struct TestAlign(Box, A>); + impl TestAlign { + fn new() -> Result { + Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?)) + } + + fn is_aligned_to(&self, align: usize) -> bool { + assert!(align.is_power_of_two()); + + let addr = self.0.as_ptr() as usize; + addr & (align - 1) == 0 + } + } + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + Ok(()) + } +} diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs new file mode 100644 index 000000000..02fda3ea5 --- /dev/null +++ b/rust/kernel/alloc/allocator/iter.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::Vmalloc; + +use crate::page; + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; + +/// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation. +/// +/// # Guarantees +/// +/// The pages iterated by the [`Iterator`] appear in the order as they are mapped in the CPU's +/// virtual address space ascendingly. +/// +/// # Invariants +/// +/// - `buf` is a valid and [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. +/// - `size` is the number of bytes from `buf` until the end of the [`Vmalloc`] allocation `buf` +/// points to. +pub struct VmallocPageIter<'a> { + /// The base address of the [`Vmalloc`] buffer. + buf: NonNull, + /// The size of the buffer pointed to by `buf` in bytes. + size: usize, + /// The current page index of the [`Iterator`]. + index: usize, + _p: PhantomData>, +} + +impl<'a> Iterator for VmallocPageIter<'a> { + type Item = page::BorrowedPage<'a>; + + fn next(&mut self) -> Option { + let offset = self.index.checked_mul(page::PAGE_SIZE)?; + + // Even though `self.size()` may be smaller than `Self::page_count() * page::PAGE_SIZE`, it + // is always a number between `(Self::page_count() - 1) * page::PAGE_SIZE` and + // `Self::page_count() * page::PAGE_SIZE`, hence the check below is sufficient. + if offset < self.size() { + self.index += 1; + } else { + return None; + } + + // SAFETY: `offset` is in the interval `[0, (self.page_count() - 1) * page::PAGE_SIZE]`, + // hence the resulting pointer is guaranteed to be within the same allocation. + let ptr = unsafe { self.buf.add(offset) }; + + // SAFETY: + // - `ptr` is a valid pointer to a `Vmalloc` allocation. + // - `ptr` is valid for the duration of `'a`. + Some(unsafe { Vmalloc::to_page(ptr) }) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.page_count().saturating_sub(self.index); + + (remaining, Some(remaining)) + } +} + +impl<'a> VmallocPageIter<'a> { + /// Creates a new [`VmallocPageIter`] instance. + /// + /// # Safety + /// + /// - `buf` must be a [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. + /// - `buf` must be valid for at least the lifetime of `'a`. + /// - `size` must be the number of bytes from `buf` until the end of the [`Vmalloc`] allocation + /// `buf` points to. + pub unsafe fn new(buf: NonNull, size: usize) -> Self { + // INVARIANT: By the safety requirements, `buf` is a valid and `page::PAGE_SIZE` aligned + // pointer into a [`Vmalloc`] allocation. + Self { + buf, + size, + index: 0, + _p: PhantomData, + } + } + + /// Returns the size of the backing [`Vmalloc`] allocation in bytes. + /// + /// Note that this is the size the [`Vmalloc`] allocation has been allocated with. Hence, this + /// number may be smaller than `[`Self::page_count`] * [`page::PAGE_SIZE`]`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Returns the number of pages owned by the backing [`Vmalloc`] allocation. + #[inline] + pub fn page_count(&self) -> usize { + self.size().div_ceil(page::PAGE_SIZE) + } +} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs new file mode 100644 index 000000000..c63d6acdb --- /dev/null +++ b/rust/kernel/alloc/kbox.rs @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Box`]. + +#[allow(unused_imports)] // Used in doc comments. +use super::allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // +}; + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::AsPageIter, + prelude::*, + types::ForeignOwnable, // +}; + +use core::{ + alloc::Layout, + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, + result::Result, // +}; + +use pin_init::ZeroableOption; + +/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. +/// +/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences, +/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not +/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`] +/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions +/// that may allocate memory are fallible. +/// +/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]. +/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]). +/// +/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KBox::::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err()); +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KVBox::::new_uninit(GFP_KERNEL).is_ok()); +/// ``` +/// +/// [`Box`]es can also be used to store trait objects by coercing their type: +/// +/// ``` +/// trait FooTrait {} +/// +/// struct FooStruct; +/// impl FooTrait for FooStruct {} +/// +/// let _ = KBox::new(FooStruct, GFP_KERNEL)? as KBox; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct Box<#[pointee] T: ?Sized, A: Allocator>(NonNull, PhantomData); + +/// Type alias for [`Box`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KBox = Box; + +/// Type alias for [`Box`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = VBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type VBox = Box; + +/// Type alias for [`Box`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KVBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KVBox = Box; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// ). +unsafe impl ZeroableOption for Box {} + +// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. +unsafe impl Send for Box +where + T: Send + ?Sized, + A: Allocator, +{ +} + +// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`. +unsafe impl Sync for Box +where + T: Sync + ?Sized, + A: Allocator, +{ +} + +impl Box +where + T: ?Sized, + A: Allocator, +{ + /// Creates a new `Box` from a raw pointer. + /// + /// # Safety + /// + /// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently + /// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the + /// `Box`. + /// + /// For ZSTs, `raw` must be a dangling, well aligned pointer. + #[inline] + pub const unsafe fn from_raw(raw: *mut T) -> Self { + // INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function. + // SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer. + Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData) + } + + /// Consumes the `Box` and returns a raw pointer. + /// + /// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive + /// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the + /// allocation, if any. + /// + /// # Examples + /// + /// ``` + /// let x = KBox::new(24, GFP_KERNEL)?; + /// let ptr = KBox::into_raw(x); + /// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`. + /// let x = unsafe { KBox::from_raw(ptr) }; + /// + /// assert_eq!(*x, 24); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn into_raw(b: Self) -> *mut T { + ManuallyDrop::new(b).0.as_ptr() + } + + /// Consumes and leaks the `Box` and returns a mutable reference. + /// + /// See [`Box::into_raw`] for more details. + #[inline] + pub fn leak<'a>(b: Self) -> &'a mut T { + // SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer + // which points to an initialized instance of `T`. + unsafe { &mut *Box::into_raw(b) } + } +} + +impl Box, A> +where + A: Allocator, +{ + /// Converts a `Box, A>` to a `Box`. + /// + /// It is undefined behavior to call this function while the value inside of `b` is not yet + /// fully initialized. + /// + /// # Safety + /// + /// Callers must ensure that the value inside of `b` is in an initialized state. + pub unsafe fn assume_init(self) -> Box { + let raw = Self::into_raw(self); + + // SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements + // of this function, the value inside the `Box` is in an initialized state. Hence, it is + // safe to reconstruct the `Box` as `Box`. + unsafe { Box::from_raw(raw.cast()) } + } + + /// Writes the value and converts to `Box`. + pub fn write(mut self, value: T) -> Box { + (*self).write(value); + + // SAFETY: We've just initialized `b`'s value. + unsafe { self.assume_init() } + } +} + +impl Box +where + A: Allocator, +{ + /// Creates a new `Box` and initializes its contents with `x`. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + pub fn new(x: T, flags: Flags) -> Result { + let b = Self::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + /// Creates a new `Box` with uninitialized contents. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::::new_uninit(GFP_KERNEL)?; + /// let b = KBox::write(b, 24); + /// + /// assert_eq!(*b, 24_u64); + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { + let layout = Layout::new::>(); + let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?; + + // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, + // which is sufficient in size and alignment for storing a `T`. + Ok(Box(ptr.cast(), PhantomData)) + } + + /// Creates a new zero-initialized `Box`. + /// + /// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in + /// which case an error is returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?; + /// assert_eq!(*b, [0; 128]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(flags: Flags) -> Result + where + T: Zeroable, + { + // SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that + // all-zeroes is a valid bit pattern for `T`. + Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() }) + } + + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then `x` will be + /// pinned in memory and can't be moved. + #[inline] + pub fn pin(x: T, flags: Flags) -> Result>, AllocError> + where + A: 'static, + { + Ok(Self::new(x, flags)?.into()) + } + + /// Construct a pinned slice of elements `Pin>`. + /// + /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks + /// or mutexes. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{ + /// new_spinlock, + /// SpinLock, // + /// }; + /// + /// struct Inner { + /// a: u32, + /// b: u32, + /// } + /// + /// #[pin_data] + /// struct Example { + /// c: u32, + /// #[pin] + /// d: SpinLock, + /// } + /// + /// impl Example { + /// fn new() -> impl PinInit { + /// try_pin_init!(Self { + /// c: 10, + /// d <- new_spinlock!(Inner { a: 20, b: 30 }), + /// }) + /// } + /// } + /// + /// // Allocate a boxed slice of 10 `Example`s. + /// let s = KBox::pin_slice( + /// | _i | Example::new(), + /// 10, + /// GFP_KERNEL + /// )?; + /// + /// assert_eq!(s[5].c, 10); + /// assert_eq!(s[3].d.lock().a, 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pin_slice( + mut init: Func, + len: usize, + flags: Flags, + ) -> Result>, E> + where + Func: FnMut(usize) -> Item, + Item: PinInit, + E: From, + { + let mut buffer = super::Vec::::with_capacity(len, flags)?; + for i in 0..len { + let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast(); + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory. + // - `ptr` is not used if an error is returned. + // - `ptr` won't be moved until it is dropped, i.e. it is pinned. + unsafe { pin_init::raw_try_init(ptr, init(i))? }; + + // SAFETY: + // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to + // `with_capacity()` above. + // - The new value at index buffer.len() + 1 is the only element being added here, and + // it has been initialized above by `raw_try_init(ptr, i)`. + unsafe { buffer.inc_len(1) }; + } + + let (ptr, _, _) = buffer.into_raw_parts(); + let slice = core::ptr::slice_from_raw_parts_mut(ptr, len); + + // SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid + // `[T]`. + Ok(Pin::from(unsafe { Box::from_raw(slice) })) + } + + /// Convert a [`Box`] to a [`Pin>`]. If `T` does not implement + /// [`Unpin`], then `x` will be pinned in memory and can't be moved. + pub fn into_pin(this: Self) -> Pin { + this.into() + } + + /// Forgets the contents (does not run the destructor), but keeps the allocation. + fn forget_contents(this: Self) -> Box, A> { + let ptr = Self::into_raw(this); + + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// let value = KBox::new([0; 32], GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = KBox::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = KBox::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drop_contents(this: Self) -> Box, A> { + let ptr = this.0.as_ptr(); + + // SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the + // value stored in `this` again. + unsafe { core::ptr::drop_in_place(ptr) }; + + Self::forget_contents(this) + } + + /// Moves the `Box`'s value out of the `Box` and consumes the `Box`. + pub fn into_inner(b: Self) -> T { + // SAFETY: By the type invariant `&*b` is valid for `read`. + let value = unsafe { core::ptr::read(&*b) }; + let _ = Self::forget_contents(b); + value + } +} + +impl From> for Pin> +where + T: ?Sized, + A: Allocator, +{ + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*b` will be pinned in memory and can't be moved. + /// + /// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory. + fn from(b: Box) -> Self { + // SAFETY: The value wrapped inside a `Pin>` cannot be moved or replaced as long + // as `T` does not implement `Unpin`. + unsafe { Pin::new_unchecked(b) } + } +} + +impl InPlaceWrite for Box, A> +where + A: Allocator + 'static, +{ + type Initialized = Box; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }.into()) + } +} + +impl InPlaceInit for Box +where + A: Allocator + 'static, +{ + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_init(init) + } +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T` allocated by `A`. +unsafe impl ForeignOwnable for Box +where + A: Allocator, +{ + const FOREIGN_ALIGN: usize = if core::mem::align_of::() < A::MIN_ALIGN { + A::MIN_ALIGN + } else { + core::mem::align_of::() + }; + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a mut T + where + Self: 'a; + + fn into_foreign(self) -> *mut c_void { + Box::into_raw(self).cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Box::from_raw(ptr.cast()) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a mut T { + let ptr = ptr.cast(); + // SAFETY: The safety requirements of this method ensure that the pointer is valid and that + // nothing else will access the value for the duration of 'a. + unsafe { &mut *ptr } + } +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T` allocated by `A`. +unsafe impl ForeignOwnable for Pin> +where + A: Allocator, +{ + const FOREIGN_ALIGN: usize = as ForeignOwnable>::FOREIGN_ALIGN; + type Borrowed<'a> + = Pin<&'a T> + where + Self: 'a; + type BorrowedMut<'a> + = Pin<&'a mut T> + where + Self: 'a; + + fn into_foreign(self) -> *mut c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Pin<&'a mut T> { + let ptr = ptr.cast(); + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &mut *ptr }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } +} + +impl Deref for Box +where + T: ?Sized, + A: Allocator, +{ + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_mut() } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for Box +where + T: ?Sized, + A: Allocator, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +impl fmt::Display for Box +where + T: ?Sized + fmt::Display, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&**self, f) + } +} + +impl fmt::Debug for Box +where + T: ?Sized + fmt::Debug, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&**self, f) + } +} + +impl Drop for Box +where + T: ?Sized, + A: Allocator, +{ + fn drop(&mut self) { + let layout = Layout::for_value::(self); + + // SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant. + unsafe { core::ptr::drop_in_place::(self.deref_mut()) }; + + // SAFETY: + // - `self.0` was previously allocated with `A`. + // - `layout` is equal to the `Layout´ `self.0` was allocated with. + unsafe { A::free(self.0.cast(), layout) }; + } +} + +/// # Examples +/// +/// ``` +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; +/// +/// let mut vbox = VBox::new((), GFP_KERNEL)?; +/// +/// assert!(vbox.page_iter().next().is_none()); +/// +/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; +/// +/// let page = vbox.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VBox { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.0.cast(); + let size = core::mem::size_of::(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs new file mode 100644 index 000000000..c7546b9da --- /dev/null +++ b/rust/kernel/alloc/kvec.rs @@ -0,0 +1,1688 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Vec`]. + +use super::{ + allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // + }, + flags::__GFP_ZERO, + layout::ArrayLayout, + AllocError, + Allocator, + Box, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::{ + AsPageIter, + PAGE_SIZE, // + }, // +}; + +use core::{ + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Index, + IndexMut, // + }, + ptr::{ + self, + NonNull, // + }, + slice::{ + self, + SliceIndex, // + }, // +}; + +use pin_init::Zeroable; + +mod errors; +pub use self::errors::{InsertError, PushError, RemoveError}; + +/// Create a [`KVec`] containing the arguments. +/// +/// New memory is allocated with `GFP_KERNEL`. +/// +/// # Examples +/// +/// ``` +/// let mut v = kernel::kvec![]; +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(v, [1]); +/// +/// let mut v = kernel::kvec![1; 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 1, 1, 4]); +/// +/// let mut v = kernel::kvec![1, 2, 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 2, 3, 4]); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[macro_export] +macro_rules! kvec { + () => ( + $crate::alloc::KVec::new() + ); + ($elem:expr; $n:expr) => ( + $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL) + ); + ($($x:expr),+ $(,)?) => ( + match $crate::alloc::KBox::new_uninit(GFP_KERNEL) { + Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))), + Err(e) => Err(e), + } + ); +} + +/// The kernel's [`Vec`] type. +/// +/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g. +/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec`. +/// +/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For +/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist. +/// +/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::`; no memory is allocated. +/// +/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the +/// capacity of the vector (the number of elements that currently fit into the vector), its length +/// (the number of elements that are currently stored in the vector) and the `Allocator` type used +/// to allocate (and free) the backing buffer. +/// +/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts +/// and manually modified. +/// +/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements +/// are added to the vector. +/// +/// # Invariants +/// +/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +/// +/// - `self.len` always represents the exact number of elements stored in the vector. +/// +/// - `self.layout` represents the absolute number of elements that can be stored within the vector +/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the +/// backing buffer to be larger than `layout`. +/// +/// - `self.len()` is always less than or equal to `self.capacity()`. +/// +/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer +/// was allocated with (and must be freed with). +pub struct Vec { + ptr: NonNull, + /// Represents the actual buffer size as `cap` times `size_of::` bytes. + /// + /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of + /// elements we can still store without reallocating. + layout: ArrayLayout, + len: usize, + _p: PhantomData, +} + +/// Type alias for [`Vec`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVec = Vec; + +/// Type alias for [`Vec`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = VVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type VVec = Vec; + +/// Type alias for [`Vec`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVVec = Vec; + +// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements. +unsafe impl Send for Vec +where + T: Send, + A: Allocator, +{ +} + +// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements. +unsafe impl Sync for Vec +where + T: Sync, + A: Allocator, +{ +} + +impl Vec +where + A: Allocator, +{ + #[inline] + const fn is_zst() -> bool { + core::mem::size_of::() == 0 + } + + /// Returns the number of elements that can be stored within the vector without allocating + /// additional memory. + pub const fn capacity(&self) -> usize { + if const { Self::is_zst() } { + usize::MAX + } else { + self.layout.len() + } + } + + /// Returns the number of elements stored within the vector. + #[inline] + pub const fn len(&self) -> usize { + self.len + } + + /// Increments `self.len` by `additional`. + /// + /// # Safety + /// + /// - `additional` must be less than or equal to `self.capacity - self.len`. + /// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized. + #[inline] + pub const unsafe fn inc_len(&mut self, additional: usize) { + // Guaranteed by the type invariant to never underflow. + debug_assert!(additional <= self.capacity() - self.len()); + // INVARIANT: By the safety requirements of this method this represents the exact number of + // elements stored within `self`. + self.len += additional; + } + + /// Decreases `self.len` by `count`. + /// + /// Returns a mutable slice to the elements forgotten by the vector. It is the caller's + /// responsibility to drop these elements if necessary. + /// + /// # Safety + /// + /// - `count` must be less than or equal to `self.len`. + unsafe fn dec_len(&mut self, count: usize) -> &mut [T] { + debug_assert!(count <= self.len()); + // INVARIANT: We relinquish ownership of the elements within the range `[self.len - count, + // self.len)`, hence the updated value of `set.len` represents the exact number of elements + // stored within `self`. + self.len -= count; + // SAFETY: The memory after `self.len()` is guaranteed to contain `count` initialized + // elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr().add(self.len), count) } + } + + /// Returns a slice of the entire vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(v.as_slice(), &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a + /// dangling raw pointer. + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self.ptr.as_ptr() + } + + /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw + /// pointer. + #[inline] + pub const fn as_ptr(&self) -> *const T { + self.ptr.as_ptr() + } + + /// Returns `true` if the vector contains no elements, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1, GFP_KERNEL); + /// assert!(!v.is_empty()); + /// ``` + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a new, empty `Vec`. + /// + /// This method does not allocate by itself. + #[inline] + pub const fn new() -> Self { + // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet, + // - `ptr` is a properly aligned dangling pointer for type `T`, + // - `layout` is an empty `ArrayLayout` (zero capacity) + // - `len` is zero, since no elements can be or have been stored, + // - `A` is always valid. + Self { + ptr: NonNull::dangling(), + layout: ArrayLayout::empty(), + len: 0, + _p: PhantomData::, + } + } + + /// Returns a slice of `MaybeUninit` for the remaining spare capacity of the vector. + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // SAFETY: + // - `self.len` is smaller than `self.capacity` by the type invariant and hence, the + // resulting pointer is guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) }.cast::>(); + + // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated + // and valid, but uninitialized. + unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) } + } + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + self.reserve(1, flags)?; + // SAFETY: The call to `reserve` was successful, so the capacity is at least one greater + // than the length. + unsafe { self.push_within_capacity_unchecked(v) }; + Ok(()) + } + + /// Appends an element to the back of the [`Vec`] instance without reallocating. + /// + /// Fails if the vector does not have capacity for the new element. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::with_capacity(10, GFP_KERNEL)?; + /// for i in 0..10 { + /// v.push_within_capacity(i)?; + /// } + /// + /// assert!(v.push_within_capacity(10).is_err()); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push_within_capacity(&mut self, v: T) -> Result<(), PushError> { + if self.len() < self.capacity() { + // SAFETY: The length is less than the capacity. + unsafe { self.push_within_capacity_unchecked(v) }; + Ok(()) + } else { + Err(PushError(v)) + } + } + + /// Appends an element to the back of the [`Vec`] instance without reallocating. + /// + /// # Safety + /// + /// The length must be less than the capacity. + unsafe fn push_within_capacity_unchecked(&mut self, v: T) { + let spare = self.spare_capacity_mut(); + + // SAFETY: By the safety requirements, `spare` is non-empty. + unsafe { spare.get_unchecked_mut(0) }.write(v); + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because the caller guarantees that + // the length is less than the capacity at the beginning of this function. + unsafe { self.inc_len(1) }; + } + + /// Inserts an element at the given index in the [`Vec`] instance. + /// + /// Fails if the vector does not have capacity for the new element. Panics if the index is out + /// of bounds. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::kvec::InsertError; + /// + /// let mut v = KVec::with_capacity(5, GFP_KERNEL)?; + /// for i in 0..5 { + /// v.insert_within_capacity(0, i)?; + /// } + /// + /// assert!(matches!(v.insert_within_capacity(0, 5), Err(InsertError::OutOfCapacity(_)))); + /// assert!(matches!(v.insert_within_capacity(1000, 5), Err(InsertError::IndexOutOfBounds(_)))); + /// assert_eq!(v, [4, 3, 2, 1, 0]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn insert_within_capacity( + &mut self, + index: usize, + element: T, + ) -> Result<(), InsertError> { + let len = self.len(); + if index > len { + return Err(InsertError::IndexOutOfBounds(element)); + } + + if len >= self.capacity() { + return Err(InsertError::OutOfCapacity(element)); + } + + // SAFETY: This is in bounds since `index <= len < capacity`. + let p = unsafe { self.as_mut_ptr().add(index) }; + // INVARIANT: This breaks the Vec invariants by making `index` contain an invalid element, + // but we restore the invariants below. + // SAFETY: Both the src and dst ranges end no later than one element after the length. + // Since the length is less than the capacity, both ranges are in bounds of the allocation. + unsafe { ptr::copy(p, p.add(1), len - index) }; + // INVARIANT: This restores the Vec invariants. + // SAFETY: The pointer is in-bounds of the allocation. + unsafe { ptr::write(p, element) }; + // SAFETY: Index `len` contains a valid element due to the above copy and write. + unsafe { self.inc_len(1) }; + Ok(()) + } + + /// Removes the last element from a vector and returns it, or `None` if it is empty. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// + /// assert_eq!(v.pop(), Some(2)); + /// assert_eq!(v.pop(), Some(1)); + /// assert_eq!(v.pop(), None); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pop(&mut self) -> Option { + if self.is_empty() { + return None; + } + + let removed: *mut T = { + // SAFETY: We just checked that the length is at least one. + let slice = unsafe { self.dec_len(1) }; + // SAFETY: The argument to `dec_len` was 1 so this returns a slice of length 1. + unsafe { slice.get_unchecked_mut(0) } + }; + + // SAFETY: The guarantees of `dec_len` allow us to take ownership of this value. + Some(unsafe { removed.read() }) + } + + /// Removes the element at the given index. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// assert_eq!(v.remove(1)?, 2); + /// assert_eq!(v, [1, 3]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn remove(&mut self, i: usize) -> Result { + let value = { + let value_ref = self.get(i).ok_or(RemoveError)?; + // INVARIANT: This breaks the invariants by invalidating the value at index `i`, but we + // restore the invariants below. + // SAFETY: The value at index `i` is valid, because otherwise we would have already + // failed with `RemoveError`. + unsafe { ptr::read(value_ref) } + }; + + // SAFETY: We checked that `i` is in-bounds. + let p = unsafe { self.as_mut_ptr().add(i) }; + + // INVARIANT: After this call, the invalid value is at the last slot, so the Vec invariants + // are restored after the below call to `dec_len(1)`. + // SAFETY: `p.add(1).add(self.len - i - 1)` is `i+1+len-i-1 == len` elements after the + // beginning of the vector, so this is in-bounds of the vector's allocation. + unsafe { ptr::copy(p.add(1), p, self.len - i - 1) }; + + // SAFETY: Since the check at the beginning of this call did not fail with `RemoveError`, + // the length is at least one. + unsafe { self.dec_len(1) }; + + Ok(value) + } + + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn with_capacity(capacity: usize, flags: Flags) -> Result { + let mut v = Vec::new(); + + v.reserve(capacity, flags)?; + + Ok(v) + } + + /// Creates a new [`Vec`] with `n` zero-initialized elements. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::zeroed(20, GFP_KERNEL)?; + /// + /// assert!(v.iter().all(|&x| x == 0)); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(n: usize, flags: Flags) -> Result + where + T: Zeroable, + { + let mut v = Self::with_capacity(n, flags | __GFP_ZERO)?; + + // SAFETY: + // - `n <= capacity - len`: `with_capacity(n)` guarantees capacity >= n, len is 0. + // - All elements in `[0, n)` are initialized: `__GFP_ZERO` zeroes the allocation, + // and `T: Zeroable` guarantees all-zeroes is a valid bit pattern. + unsafe { v.inc_len(n) }; + Ok(v) + } + + /// Creates a `Vec` from a pointer, a length and a capacity using the allocator `A`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.reserve(1, GFP_KERNEL)?; + /// + /// let (mut ptr, mut len, cap) = v.into_raw_parts(); + /// + /// // SAFETY: We've just reserved memory for another element. + /// unsafe { ptr.add(len).write(4) }; + /// len += 1; + /// + /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and + /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it + /// // from the exact same raw parts. + /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) }; + /// + /// assert_eq!(v, [1, 2, 3, 4]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// If `T` is a ZST: + /// + /// - `ptr` must be a dangling, well aligned pointer. + /// + /// Otherwise: + /// + /// - `ptr` must have been allocated with the allocator `A`. + /// - `ptr` must satisfy or exceed the alignment requirements of `T`. + /// - `ptr` must point to memory with a size of at least `size_of::() * capacity` bytes. + /// - The allocated size in bytes must not be larger than `isize::MAX`. + /// - `length` must be less than or equal to `capacity`. + /// - The first `length` elements must be initialized values of type `T`. + /// + /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for + /// `cap` and `len`. + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + let layout = if Self::is_zst() { + ArrayLayout::empty() + } else { + // SAFETY: By the safety requirements of this function, `capacity * size_of::()` is + // smaller than `isize::MAX`. + unsafe { ArrayLayout::new_unchecked(capacity) } + }; + + // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are + // covered by the safety requirements of this function. + Self { + // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid + // memory allocation, allocated with `A`. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + layout, + len: length, + _p: PhantomData::, + } + } + + /// Consumes the `Vec` and returns its raw components `pointer`, `length` and `capacity`. + /// + /// This will not run the destructor of the contained elements and for non-ZSTs the allocation + /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the + /// elements and free the allocation, if any. + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + (ptr, len, capacity) + } + + /// Clears the vector, removing all values. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// + /// v.clear(); + /// + /// assert!(v.is_empty()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn clear(&mut self) { + self.truncate(0); + } + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= v.len() + 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if Self::is_zst() { + // The capacity is already `usize::MAX` for ZSTs, we can't go higher. + return Err(AllocError); + } + + // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the + // multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `ptr` is valid because it's either `None` or comes from a previous call to + // `A::realloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + A::realloc( + Some(self.ptr.cast()), + layout.into(), + self.layout.into(), + flags, + NumaNode::NO_NODE, + )? + }; + + // INVARIANT: + // - `layout` is some `ArrayLayout::`, + // - `ptr` has been created by `A::realloc` from `layout`. + self.ptr = ptr.cast(); + self.layout = layout; + + Ok(()) + } + + /// Shortens the vector, setting the length to `len` and drops the removed values. + /// If `len` is greater than or equal to the current length, this does nothing. + /// + /// This has no effect on the capacity and will not allocate. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.truncate(1); + /// assert_eq!(v.len(), 1); + /// assert_eq!(&v, &[1]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn truncate(&mut self, len: usize) { + if let Some(count) = self.len().checked_sub(len) { + // SAFETY: `count` is `self.len() - len` so it is guaranteed to be less than or + // equal to `self.len()`. + let ptr: *mut [T] = unsafe { self.dec_len(count) }; + + // SAFETY: the contract of `dec_len` guarantees that the elements in `ptr` are + // valid elements whose ownership has been transferred to the caller. + unsafe { ptr::drop_in_place(ptr) }; + } + } + + /// Takes ownership of all items in this vector without consuming the allocation. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![0, 1, 2, 3]?; + /// + /// for (i, j) in v.drain_all().enumerate() { + /// assert_eq!(i, j); + /// } + /// + /// assert!(v.capacity() >= 4); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drain_all(&mut self) -> DrainAll<'_, T> { + // SAFETY: This does not underflow the length. + let elems = unsafe { self.dec_len(self.len()) }; + // INVARIANT: The first `len` elements of the spare capacity are valid values, and as we + // just set the length to zero, we may transfer ownership to the `DrainAll` object. + DrainAll { + elements: elems.iter_mut(), + } + } + + /// Removes all elements that don't match the provided closure. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3, 4]?; + /// v.retain(|i| *i % 2 == 0); + /// assert_eq!(v, [2, 4]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn retain(&mut self, mut f: impl FnMut(&mut T) -> bool) { + let mut num_kept = 0; + let mut next_to_check = 0; + while let Some(to_check) = self.get_mut(next_to_check) { + if f(to_check) { + self.swap(num_kept, next_to_check); + num_kept += 1; + } + next_to_check += 1; + } + self.truncate(num_kept); + } +} +// TODO: This is a temporary KVVec-specific implementation. It should be replaced with a generic +// `shrink_to()` for `impl Vec` that uses `A::realloc()` once the +// underlying allocators properly support shrinking via realloc. +impl Vec { + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// For `kmalloc` allocations, this delegates to `realloc()`, which decides whether + /// shrinking is worthwhile. For `vmalloc` allocations, shrinking only occurs if the + /// operation would free at least one page of memory, and performs a deep copy since + /// `vrealloc` does not yet support in-place shrinking. + /// + /// # Examples + /// + /// ``` + /// // Allocate enough capacity to span multiple pages. + /// let elements_per_page = kernel::page::PAGE_SIZE / core::mem::size_of::(); + /// let mut v = KVVec::with_capacity(elements_per_page * 4, GFP_KERNEL)?; + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// + /// v.shrink_to(0, GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn shrink_to(&mut self, min_capacity: usize, flags: Flags) -> Result<(), AllocError> { + let target_cap = core::cmp::max(self.len(), min_capacity); + + if self.capacity() <= target_cap { + return Ok(()); + } + + if Self::is_zst() { + return Ok(()); + } + + // For kmalloc allocations, delegate to realloc() and let the allocator decide + // whether shrinking is worthwhile. + // + // SAFETY: `self.ptr` points to a valid `KVmalloc` allocation. + if !unsafe { bindings::is_vmalloc_addr(self.ptr.as_ptr().cast()) } { + let new_layout = ArrayLayout::::new(target_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `self.ptr` is valid and was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + KVmalloc::realloc( + Some(self.ptr.cast()), + new_layout.into(), + self.layout.into(), + flags, + NumaNode::NO_NODE, + )? + }; + + self.ptr = ptr.cast(); + self.layout = new_layout; + return Ok(()); + } + + // Only shrink if we would free at least one page. + let current_size = self.capacity() * core::mem::size_of::(); + let target_size = target_cap * core::mem::size_of::(); + let current_pages = current_size.div_ceil(PAGE_SIZE); + let target_pages = target_size.div_ceil(PAGE_SIZE); + + if current_pages <= target_pages { + return Ok(()); + } + + if target_cap == 0 { + if !self.layout.is_empty() { + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + } + self.ptr = NonNull::dangling(); + self.layout = ArrayLayout::empty(); + return Ok(()); + } + + // SAFETY: `target_cap <= self.capacity()` and original capacity was valid. + let new_layout = unsafe { ArrayLayout::::new_unchecked(target_cap) }; + + let new_ptr = KVmalloc::alloc(new_layout.into(), flags, NumaNode::NO_NODE)?; + + // SAFETY: + // - `self.as_ptr()` is valid for reads of `self.len()` elements of `T`. + // - `new_ptr` is valid for writes of at least `target_cap >= self.len()` elements. + // - The two allocations do not overlap since `new_ptr` is freshly allocated. + // - Both pointers are properly aligned for `T`. + unsafe { + ptr::copy_nonoverlapping(self.as_ptr(), new_ptr.as_ptr().cast::(), self.len()) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + + self.ptr = new_ptr.cast::(); + self.layout = new_layout; + + Ok(()) + } +} + +impl Vec { + /// Extend the vector by `n` clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_with(3, 5, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5]); + /// + /// v.extend_with(2, 8, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// v.extend_with(0, 3, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { + if n == 0 { + return Ok(()); + } + + self.reserve(n, flags)?; + + let spare = self.spare_capacity_mut(); + + for item in spare.iter_mut().take(n - 1) { + item.write(value.clone()); + } + + // We can write the last element directly without cloning needlessly. + spare[n - 1].write(value); + + // SAFETY: + // - `self.len() + n <= self.capacity()` due to the call to reserve above, + // - the loop and the line above initialized the next `n` elements. + unsafe { self.inc_len(n) }; + + Ok(()) + } + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> { + self.reserve(other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: + // - `other.len()` spare entries have just been initialized, so it is safe to increase + // the length by the same number. + // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve` + // call. + unsafe { self.inc_len(other.len()) }; + Ok(()) + } + + /// Create a new `Vec` and extend it by `n` clones of `value`. + pub fn from_elem(value: T, n: usize, flags: Flags) -> Result { + let mut v = Self::with_capacity(n, flags)?; + + v.extend_with(n, value, flags)?; + + Ok(v) + } + + /// Resizes the [`Vec`] so that `len` is equal to `new_len`. + /// + /// If `new_len` is smaller than `len`, the `Vec` is [`Vec::truncate`]d. + /// If `new_len` is larger, each new slot is filled with clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.resize(1, 42, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.resize(3, 42, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 42, 42]); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn resize(&mut self, new_len: usize, value: T, flags: Flags) -> Result<(), AllocError> { + match new_len.checked_sub(self.len()) { + Some(n) => self.extend_with(n, value, flags), + None => { + self.truncate(new_len); + Ok(()) + } + } + } +} + +impl Drop for Vec +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant. + unsafe { + ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr(), + self.len, + )) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.ptr.cast(), self.layout.into()) }; + } +} + +impl From> for Vec +where + A: Allocator, +{ + fn from(b: Box<[T; N], A>) -> Vec { + let len = b.len(); + let ptr = Box::into_raw(b); + + // SAFETY: + // - `b` has been allocated with `A`, + // - `ptr` fulfills the alignment requirements for `T`, + // - `ptr` points to memory with at least a size of `size_of::() * len`, + // - all elements within `b` are initialized values of `T`, + // - `len` does not exceed `isize::MAX`. + unsafe { Vec::from_raw_parts(ptr.cast(), len, len) } + } +} + +impl Default for Vec { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Deref for Vec +where + A: Allocator, +{ + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +impl DerefMut for Vec +where + A: Allocator, +{ + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// struct Foo>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow<[T]> for Vec +where + A: Allocator, +{ + fn borrow(&self) -> &[T] { + self.as_slice() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// struct Foo>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let mut arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&mut arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut<[T]> for Vec +where + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Eq for Vec where A: Allocator {} + +impl, A> Index for Vec +where + A: Allocator, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +impl, A> IndexMut for Vec +where + A: Allocator, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +macro_rules! impl_slice_eq { + ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => { + $( + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + } + )* + } +} + +impl_slice_eq! { + [A1: Allocator, A2: Allocator] Vec, Vec, + [A: Allocator] Vec, &[U], + [A: Allocator] Vec, &mut [U], + [A: Allocator] &[T], Vec, + [A: Allocator] &mut [T], Vec, + [A: Allocator] Vec, [U], + [A: Allocator] [T], Vec, + [A: Allocator, const N: usize] Vec, [U; N], + [A: Allocator, const N: usize] Vec, &[U; N], +} + +impl<'a, T, A> IntoIterator for &'a Vec +where + A: Allocator, +{ + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec +where + A: Allocator, +{ + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// # Examples +/// +/// ``` +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; +/// +/// let mut vec = VVec::::new(); +/// +/// assert!(vec.page_iter().next().is_none()); +/// +/// vec.reserve(PAGE_SIZE, GFP_KERNEL)?; +/// +/// let page = vec.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VVec { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.ptr.cast(); + let size = self.layout.size(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} + +/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. +/// +/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the +/// [`IntoIterator`] trait). +/// +/// # Examples +/// +/// ``` +/// let v = kernel::kvec![0, 1, 2]?; +/// let iter = v.into_iter(); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct IntoIter { + ptr: *mut T, + buf: NonNull, + len: usize, + layout: ArrayLayout, + _p: PhantomData, +} + +impl IntoIter +where + A: Allocator, +{ + fn into_raw_parts(self) -> (*mut T, NonNull, usize, usize) { + let me = ManuallyDrop::new(self); + let ptr = me.ptr; + let buf = me.buf; + let len = me.len; + let cap = me.layout.len(); + (ptr, buf, len, cap) + } + + /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`. + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// + /// let v = it.collect(GFP_KERNEL); + /// assert_eq!(v, [2, 3]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Implementation details + /// + /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait + /// in the kernel, namely: + /// + /// - Rust's specialization feature is unstable. This prevents us to optimize for the special + /// case where `I::IntoIter` equals `Vec`'s `IntoIter` type. + /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` + /// doesn't require this type to be `'static`. + /// - `FromIterator::from_iter` does return `Self` instead of `Result`, hence + /// we can't properly handle allocation failures. + /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation + /// flags. + /// + /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a + /// `Vec` again. + /// + /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing + /// buffer. However, this backing buffer may be shrunk to the actual count of elements. + pub fn collect(self, flags: Flags) -> Vec { + let old_layout = self.layout; + let (mut ptr, buf, len, mut cap) = self.into_raw_parts(); + let has_advanced = ptr != buf.as_ptr(); + + if has_advanced { + // Copy the contents we have advanced to at the beginning of the buffer. + // + // SAFETY: + // - `ptr` is valid for reads of `len * size_of::()` bytes, + // - `buf.as_ptr()` is valid for writes of `len * size_of::()` bytes, + // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to + // each other, + // - both `ptr` and `buf.ptr()` are properly aligned. + unsafe { ptr::copy(ptr, buf.as_ptr(), len) }; + ptr = buf.as_ptr(); + + // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()` by the type + // invariant. + let layout = unsafe { ArrayLayout::::new_unchecked(len) }; + + // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed by + // the type invariant to be smaller than `cap`. Depending on `realloc` this operation + // may shrink the buffer or leave it as it is. + ptr = match unsafe { + A::realloc( + Some(buf.cast()), + layout.into(), + old_layout.into(), + flags, + NumaNode::NO_NODE, + ) + } { + // If we fail to shrink, which likely can't even happen, continue with the existing + // buffer. + Err(_) => ptr, + Ok(ptr) => { + cap = len; + ptr.as_ptr().cast() + } + }; + } + + // SAFETY: If the iterator has been advanced, the advanced elements have been copied to + // the beginning of the buffer and `len` has been adjusted accordingly. + // + // - `ptr` is guaranteed to point to the start of the backing buffer. + // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`. + // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original + // `Vec`. + unsafe { Vec::from_raw_parts(ptr, len, cap) } + } +} + +impl Iterator for IntoIter +where + A: Allocator, +{ + type Item = T; + + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// assert_eq!(it.next(), Some(2)); + /// assert_eq!(it.next(), Some(3)); + /// assert_eq!(it.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn next(&mut self) -> Option { + if self.len == 0 { + return None; + } + + let current = self.ptr; + + // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr` + // by one guarantees that. + unsafe { self.ptr = self.ptr.add(1) }; + + self.len -= 1; + + // SAFETY: `current` is guaranteed to point at a valid element within the buffer. + Some(unsafe { current.read() }) + } + + /// # Examples + /// + /// ``` + /// let v: KVec = kernel::kvec![1, 2, 3]?; + /// let mut iter = v.into_iter(); + /// let size = iter.size_hint().0; + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 1); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 2); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 3); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl Drop for IntoIter +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) }; + + // SAFETY: + // - `self.buf` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.buf.cast(), self.layout.into()) }; + } +} + +impl IntoIterator for Vec +where + A: Allocator, +{ + type Item = T; + type IntoIter = IntoIter; + + /// Consumes the `Vec` and creates an `Iterator`, which moves each value out of the + /// vector (from start to end). + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2]?; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some(1)); + /// assert_eq!(v_iter.next(), Some(2)); + /// assert_eq!(v_iter.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// let v = kernel::kvec![]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, None); + /// + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + let buf = self.ptr; + let layout = self.layout; + let (ptr, len, _) = self.into_raw_parts(); + + IntoIter { + ptr, + buf, + len, + layout, + _p: PhantomData::, + } + } +} + +/// An iterator that owns all items in a vector, but does not own its allocation. +/// +/// # Invariants +/// +/// Every `&mut T` returned by the iterator references a `T` that the iterator may take ownership +/// of. +pub struct DrainAll<'vec, T> { + elements: slice::IterMut<'vec, T>, +} + +impl<'vec, T> Iterator for DrainAll<'vec, T> { + type Item = T; + + fn next(&mut self) -> Option { + let elem: *mut T = self.elements.next()?; + // SAFETY: By the type invariants, we may take ownership of this value. + Some(unsafe { elem.read() }) + } + + fn size_hint(&self) -> (usize, Option) { + self.elements.size_hint() + } +} + +impl<'vec, T> Drop for DrainAll<'vec, T> { + fn drop(&mut self) { + if core::mem::needs_drop::() { + let iter = core::mem::take(&mut self.elements); + let ptr: *mut [T] = iter.into_slice(); + // SAFETY: By the type invariants, we own these values so we may destroy them. + unsafe { ptr::drop_in_place(ptr) }; + } + } +} + +#[cfg(CONFIG_RUST_KVEC_KUNIT_TEST)] +#[macros::kunit_tests(rust_kvec)] +mod tests { + use super::*; + use crate::prelude::*; + + #[test] + fn test_kvec_retain() { + /// Verify correctness for one specific function. + #[expect(clippy::needless_range_loop)] + fn verify(c: &[bool]) { + let mut vec1: KVec = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap(); + let mut vec2: KVec = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap(); + + for i in 0..c.len() { + vec1.push_within_capacity(i).unwrap(); + if c[i] { + vec2.push_within_capacity(i).unwrap(); + } + } + + vec1.retain(|i| c[*i]); + + assert_eq!(vec1, vec2); + } + + /// Add one to a binary integer represented as a boolean array. + fn add(value: &mut [bool]) { + let mut carry = true; + for v in value { + let new_v = carry != *v; + carry = carry && *v; + *v = new_v; + } + } + + // This boolean array represents a function from index to boolean. We check that `retain` + // behaves correctly for all possible boolean arrays of every possible length less than + // ten. + let mut func = KVec::with_capacity(10, GFP_KERNEL).unwrap(); + for len in 0..10 { + for _ in 0u32..1u32 << len { + verify(&func); + add(&mut func); + } + func.push_within_capacity(false).unwrap(); + } + } + + #[test] + fn test_kvvec_shrink_to() { + use crate::page::PAGE_SIZE; + + // Create a vector with capacity spanning multiple pages. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add a few elements. + v.push(1, GFP_KERNEL).unwrap(); + v.push(2, GFP_KERNEL).unwrap(); + v.push(3, GFP_KERNEL).unwrap(); + + let initial_capacity = v.capacity(); + assert!(initial_capacity >= PAGE_SIZE * 4); + + // Shrink to a capacity that would free at least one page. + v.shrink_to(PAGE_SIZE, GFP_KERNEL).unwrap(); + + // Capacity should have been reduced. + assert!(v.capacity() < initial_capacity); + assert!(v.capacity() >= PAGE_SIZE); + + // Elements should be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + + // Shrink to zero (should shrink to len). + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Capacity should be at least the length. + assert!(v.capacity() >= v.len()); + + // Elements should still be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + } + + #[test] + fn test_kvvec_shrink_to_empty() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity but no elements. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + assert!(v.is_empty()); + + // Shrink empty vector to zero. + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Should have freed the allocation. + assert_eq!(v.capacity(), 0); + assert!(v.is_empty()); + } + + #[test] + fn test_kvvec_shrink_to_no_op() { + use crate::page::PAGE_SIZE; + + // Create a small vector. + let mut v = KVVec::::with_capacity(PAGE_SIZE, GFP_KERNEL).unwrap(); + v.push(1, GFP_KERNEL).unwrap(); + + let capacity_before = v.capacity(); + + // Try to shrink to a capacity larger than current - should be no-op. + v.shrink_to(capacity_before + 100, GFP_KERNEL).unwrap(); + + assert_eq!(v.capacity(), capacity_before); + assert_eq!(v.len(), 1); + assert_eq!(v[0], 1); + } + + #[test] + fn test_kvvec_shrink_to_respects_min_capacity() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity. + let mut v = KVVec::::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add some elements. + for i in 0..10u8 { + v.push(i, GFP_KERNEL).unwrap(); + } + + // Shrink to a min_capacity larger than length. + let min_cap = PAGE_SIZE * 2; + v.shrink_to(min_cap, GFP_KERNEL).unwrap(); + + // Capacity should be at least min_capacity. + assert!(v.capacity() >= min_cap); + + // All elements preserved. + assert_eq!(v.len(), 10); + for i in 0..10u8 { + assert_eq!(v[i as usize], i); + } + } +} diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs new file mode 100644 index 000000000..aaca64465 --- /dev/null +++ b/rust/kernel/alloc/kvec/errors.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Errors for the [`Vec`] type. + +use crate::{ + fmt, + prelude::*, // +}; + +/// Error type for [`Vec::push_within_capacity`]. +pub struct PushError(pub T); + +impl fmt::Debug for PushError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Not enough capacity") + } +} + +impl From> for Error { + #[inline] + fn from(_: PushError) -> Error { + // Returning ENOMEM isn't appropriate because the system is not out of memory. The vector + // is just full and we are refusing to resize it. + EINVAL + } +} + +/// Error type for [`Vec::remove`]. +pub struct RemoveError; + +impl fmt::Debug for RemoveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Index out of bounds") + } +} + +impl From for Error { + #[inline] + fn from(_: RemoveError) -> Error { + EINVAL + } +} + +/// Error type for [`Vec::insert_within_capacity`]. +pub enum InsertError { + /// The value could not be inserted because the index is out of bounds. + IndexOutOfBounds(T), + /// The value could not be inserted because the vector is out of capacity. + OutOfCapacity(T), +} + +impl fmt::Debug for InsertError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + InsertError::IndexOutOfBounds(_) => write!(f, "Index out of bounds"), + InsertError::OutOfCapacity(_) => write!(f, "Not enough capacity"), + } + } +} + +impl From> for Error { + #[inline] + fn from(_: InsertError) -> Error { + EINVAL + } +} diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs new file mode 100644 index 000000000..62a459c66 --- /dev/null +++ b/rust/kernel/alloc/layout.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory layout. +//! +//! Custom layout types extending or improving [`Layout`]. + +use core::{ + alloc::Layout, + marker::PhantomData, // +}; + +/// Error when constructing an [`ArrayLayout`]. +pub struct LayoutError; + +/// A layout for an array `[T; n]`. +/// +/// # Invariants +/// +/// - `len * size_of::() <= isize::MAX`. +pub struct ArrayLayout { + len: usize, + _phantom: PhantomData T>, +} + +impl Clone for ArrayLayout { + fn clone(&self) -> Self { + *self + } +} +impl Copy for ArrayLayout {} + +const ISIZE_MAX: usize = isize::MAX as usize; + +impl ArrayLayout { + /// Creates a new layout for `[T; 0]`. + pub const fn empty() -> Self { + // INVARIANT: `0 * size_of::() <= isize::MAX`. + Self { + len: 0, + _phantom: PhantomData, + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Errors + /// + /// When `len * size_of::()` overflows or when `len * size_of::() > isize::MAX`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::alloc::layout::{ + /// # ArrayLayout, + /// # LayoutError, // + /// # }; + /// let layout = ArrayLayout::::new(15)?; + /// assert_eq!(layout.len(), 15); + /// + /// // Errors because `len * size_of::()` overflows. + /// let layout = ArrayLayout::::new(isize::MAX as usize); + /// assert!(layout.is_err()); + /// + /// // Errors because `len * size_of::() > isize::MAX`, + /// // even though `len < isize::MAX`. + /// let layout = ArrayLayout::::new(isize::MAX as usize / 2); + /// assert!(layout.is_err()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub const fn new(len: usize) -> Result { + match len.checked_mul(core::mem::size_of::()) { + Some(size) if size <= ISIZE_MAX => { + // INVARIANT: We checked above that `len * size_of::() <= isize::MAX`. + Ok(Self { + len, + _phantom: PhantomData, + }) + } + _ => Err(LayoutError), + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Safety + /// + /// `len` must be a value, for which `len * size_of::() <= isize::MAX` is true. + pub const unsafe fn new_unchecked(len: usize) -> Self { + // INVARIANT: By the safety requirements of this function + // `len * size_of::() <= isize::MAX`. + Self { + len, + _phantom: PhantomData, + } + } + + /// Returns the number of array elements represented by this layout. + pub const fn len(&self) -> usize { + self.len + } + + /// Returns `true` when no array elements are represented by this layout. + pub const fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the size of the [`ArrayLayout`] in bytes. + pub const fn size(&self) -> usize { + self.len() * core::mem::size_of::() + } +} + +impl From> for Layout { + fn from(value: ArrayLayout) -> Self { + let res = Layout::array::(value.len); + // SAFETY: By the type invariant of `ArrayLayout` we have + // `len * size_of::() <= isize::MAX` and thus the result must be `Ok`. + unsafe { res.unwrap_unchecked() } + } +} diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs new file mode 100644 index 000000000..60dfbec8f --- /dev/null +++ b/rust/kernel/auxiliary.rs @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the auxiliary bus. +//! +//! C header: [`include/linux/auxiliary_bus.h`](srctree/include/linux/auxiliary_bus.h) + +use crate::{ + bindings, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // + }, + ThisModule, // +}; +use core::{ + any::TypeId, + marker::PhantomData, + mem::offset_of, + pin::Pin, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +/// An adapter for the registration of auxiliary drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::auxiliary_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct auxiliary_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::auxiliary_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + adrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct auxiliary_driver` on initialization. + unsafe { + (*adrv.get()).name = name.as_char_ptr(); + (*adrv.get()).probe = Some(Self::probe_callback); + (*adrv.get()).remove = Some(Self::remove_callback); + (*adrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__auxiliary_driver_register(adrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(adrv: &Opaque) { + // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::auxiliary_driver_unregister(adrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback( + adev: *mut bindings::auxiliary_device, + id: *const bindings::auxiliary_device_id, + ) -> c_int { + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a + // `struct auxiliary_device`. + // + // INVARIANT: `adev` is valid for the duration of `probe_callback()`. + let adev = unsafe { &*adev.cast::>>() }; + + // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. + let info = unsafe { id.info_unchecked::() }; + + from_result(|| { + let data = T::probe(adev, info); + + adev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a + // `struct auxiliary_device`. + // + // INVARIANT: `adev` is valid for the duration of `remove_callback()`. + let adev = unsafe { &*adev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { adev.as_ref().drvdata_borrow::>() }; + + T::unbind(adev, data); + } +} + +/// Declares a kernel module that exposes a single auxiliary driver. +#[macro_export] +macro_rules! module_auxiliary_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::auxiliary::Adapter, { $($f)* }); + }; +} + +/// Abstraction for `bindings::auxiliary_device_id`. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::auxiliary_device_id); + +impl DeviceId { + /// Create a new [`DeviceId`] from name. + pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self { + let name = name.to_bytes_with_nul(); + let modname = modname.to_bytes_with_nul(); + + let mut id: bindings::auxiliary_device_id = pin_init::zeroed(); + let mut i = 0; + while i < modname.len() { + id.name[i] = modname[i]; + i += 1; + } + + // Reuse the space of the NULL terminator. + id.name[i - 1] = b'.'; + + let mut j = 0; + while j < name.len() { + id.name[i] = name[j]; + i += 1; + j += 1; + } + + Self(id) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `auxiliary_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::auxiliary_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = + core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); +} + +/// IdTable type for auxiliary drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a auxiliary `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! auxiliary_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("auxiliary", $crate::auxiliary::DeviceId, $($tt)*); + }; +} + +/// The auxiliary driver trait. +/// +/// Drivers must implement this trait in order to get an auxiliary driver registered. +pub trait Driver { + /// The type holding information about each device id supported by the driver. + /// + /// TODO: Use associated_type_defaults once stabilized: + /// + /// type IdInfo: 'static = (); + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// Auxiliary driver probe. + /// + /// Called when an auxiliary device is matches a corresponding driver. + fn probe<'bound>( + dev: &'bound Device>, + id_info: &'bound Self::IdInfo, + ) -> impl PinInit, Error> + 'bound; + + /// Auxiliary driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The auxiliary device representation. +/// +/// This structure represents the Rust abstraction for a C `struct auxiliary_device`. The +/// implementation abstracts the usage of an already existing C `struct auxiliary_device` within +/// Rust code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct auxiliary_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::auxiliary_device { + self.0.get() + } + + /// Returns the auxiliary device' id. + pub fn id(&self) -> u32 { + // SAFETY: By the type invariant `self.as_raw()` is a valid pointer to a + // `struct auxiliary_device`. + unsafe { (*self.as_raw()).id } + } +} + +impl Device { + /// Returns a bound reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device { + let parent = (**self).parent(); + + // SAFETY: A bound auxiliary device always has a bound parent device. + unsafe { parent.as_bound() } + } + + /// Returns the stored registration data as a pinned reference. + /// + /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`]. + /// + /// # Safety + /// + /// Callers must ensure that the lifetime shortening from the original `'static` storage to + /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee. + unsafe fn registration_data_pinned(&self) -> Result>> { + // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. + let ptr = unsafe { (*self.as_raw()).registration_data_rust }; + if ptr.is_null() { + dev_warn!( + self.as_ref(), + "No registration data set; parent is not a Rust driver.\n" + ); + return Err(ENOENT); + } + + // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`; + // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId` + // at the start of the allocation is valid regardless of `F`. + let type_id = unsafe { ptr.cast::().read() }; + if type_id != TypeId::of::() { + return Err(EINVAL); + } + + // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s + // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is + // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until + // `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::>>>::borrow(ptr) }; + + // SAFETY: `data` is a structurally pinned field of `RegistrationData`. + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } + + /// Access the registration data set by the registering (parent) driver through a closure. + /// + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned + /// reference to the registration data. + /// + /// For covariant types that implement [`trait@CovariantForLt`], prefer + /// [`registration_data`](Self::registration_data) which returns a direct reference. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data_with( + &self, + f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + ) -> Result { + // SAFETY: The HRTB closure prevents the caller from smuggling in references with a + // concrete short lifetime, making the round-trip from `'static` sound regardless of + // variance. + let pinned = unsafe { self.registration_data_pinned::()? }; + + Ok(f(pinned)) + } + + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees + /// that the lifetime shortening is sound. + /// + /// For non-covariant types, use the closure-based [`Self::registration_data_with`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data(&self) -> Result>> { + // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening + // from `'static` to `'_` performed by `registration_data_pinned` sound. + unsafe { self.registration_data_pinned::() } + } +} + +impl Device { + /// Returns a reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device { + // SAFETY: A `struct auxiliary_device` always has a parent. + unsafe { self.as_ref().parent().unwrap_unchecked() } + } + + extern "C" fn release(dev: *mut bindings::device) { + // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device` + // embedded in `struct auxiliary_device`. + let adev = unsafe { container_of!(dev, bindings::auxiliary_device, dev) }; + + // SAFETY: `adev` points to the memory that has been allocated in `Registration::new`, via + // `KBox::new(Opaque::::zeroed(), GFP_KERNEL)`. + let _ = unsafe { KBox::>::from_raw(adev.cast()) }; + } +} + +// SAFETY: `auxiliary::Device` is a transparent wrapper of `struct auxiliary_device`. +// The offset is guaranteed to point to a valid device field inside `auxiliary::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::auxiliary_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // CAST: `Self` a transparent wrapper of `bindings::auxiliary_device`. + let adev: *mut bindings::auxiliary_device = obj.cast().as_ptr(); + + // SAFETY: By the type invariant of `Self`, `adev` is a pointer to a valid + // `struct auxiliary_device`. + let dev = unsafe { addr_of_mut!((*adev).dev) }; + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(dev) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct auxiliary_device`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct auxiliary_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking. +#[repr(C)] +#[pin_data] +struct RegistrationData { + type_id: TypeId, + #[pin] + data: T, +} + +/// The registration of an auxiliary device. +/// +/// This type represents the registration of a [`struct auxiliary_device`]. When its parent device +/// is unbound, the corresponding auxiliary device will be unregistered from the system. +/// +/// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration +/// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). +/// +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and +/// [`Device::registration_data_with()`]. +/// +/// # Invariants +/// +/// `self.adev` always holds a valid pointer to an initialized and registered +/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a +/// valid `Pin>>>`. +pub struct Registration<'a, F: ForLt + 'static> { + adev: NonNull, + _phantom: PhantomData>, +} + +impl<'a, F: ForLt> Registration<'a, F> +where + for<'b> F::Of<'b>: Send + Sync, +{ + /// Create and register a new auxiliary device with the given registration data. + /// + /// The `data` is owned by the registration and can be accessed through the auxiliary device + /// via [`Device::registration_data()`]. + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + /// + /// If the registration data is `'static`, use the safe [`Registration::new()`] instead. + pub unsafe fn new_with_lt( + parent: &'a device::Device, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit, E>, + ) -> Result + where + Error: From, + { + let data = KBox::pin_init::( + try_pin_init!(RegistrationData { + type_id: TypeId::of::(), + data <- data, + }), + GFP_KERNEL, + )?; + + // SAFETY: `'a` is invariant (via `Registration`'s `PhantomData`). Lifetimes do not + // affect layout, so RegistrationData> and RegistrationData> + // have identical representation. + let data: Pin>>> = + unsafe { core::mem::transmute(data) }; + + let boxed: KBox> = KBox::zeroed(GFP_KERNEL)?; + let adev = boxed.get(); + + // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. + unsafe { + (*adev).dev.parent = parent.as_raw(); + (*adev).dev.release = Some(Device::release); + (*adev).name = name.as_char_ptr(); + (*adev).id = id; + (*adev).registration_data_rust = data.into_foreign(); + } + + // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, + // which has not been initialized yet. + unsafe { bindings::auxiliary_device_init(adev) }; + + // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be + // freed by `Device::release` when the last reference to the `struct auxiliary_device` + // is dropped. + let _ = KBox::into_raw(boxed); + + // SAFETY: + // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which + // has been initialized, + // - `modname.as_char_ptr()` is a NULL terminated string. + let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; + if ret != 0 { + // SAFETY: `registration_data` was set above via `into_foreign()`. + drop(unsafe { + Pin::>>>::from_foreign( + (*adev).registration_data_rust, + ) + }); + + // SAFETY: `adev` is guaranteed to be a valid pointer to a + // `struct auxiliary_device`, which has been initialized. + unsafe { bindings::auxiliary_device_uninit(adev) }; + + return Err(Error::from_errno(ret)); + } + + // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is + // called, which happens in `Self::drop()`. + Ok(Self { + // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated + // successfully. + adev: unsafe { NonNull::new_unchecked(adev) }, + _phantom: PhantomData, + }) + } + + /// Create and register a new auxiliary device with `'static` registration data. + /// + /// Safe variant of [`Registration::new_with_lt()`] for registration data that does not contain + /// borrowed references. + pub fn new( + parent: &'a device::Device, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit, E>, + ) -> Result + where + F::Of<'a>: 'static, + Error: From, + { + // SAFETY: `F::Of<'a>: 'static` guarantees the data contains no borrowed references, + // so forgetting the `Registration` cannot cause use-after-free. + unsafe { Self::new_with_lt(parent, name, id, modname, data) } + } +} + +impl Drop for Registration<'_, F> { + fn drop(&mut self) { + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered + // `struct auxiliary_device`. + unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) }; + + // SAFETY: `registration_data` was set in `new()` via `into_foreign()`. + drop(unsafe { + Pin::>>>::from_foreign( + (*self.adev.as_ptr()).registration_data_rust, + ) + }); + + // This drops the reference we acquired through `auxiliary_device_init()`. + // + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered + // `struct auxiliary_device`. + unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) }; + } +} + +// SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread. +unsafe impl Send for Registration<'_, F> where for<'a> F::Of<'a>: Send {} + +// SAFETY: `Registration` does not expose any methods or fields that need synchronization. +unsafe impl Sync for Registration<'_, F> where for<'a> F::Of<'a>: Send {} diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs new file mode 100644 index 000000000..a0d089423 --- /dev/null +++ b/rust/kernel/bitfield.rs @@ -0,0 +1,865 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for defining bitfields as Rust structures. +//! +//! The [`bitfield!`](kernel::bitfield!) macro declares integer types that are split into distinct +//! bit fields of arbitrary length. Each field is typed using [`Bounded`](kernel::num::Bounded) to +//! ensure values are properly validated and to avoid implicit data loss. +//! +//! # Example +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! bitfield! { +//! pub struct Rgb(u16) { +//! 15:11 blue; +//! 10:5 green; +//! 4:0 red; +//! } +//! } +//! +//! // Valid value for the `blue` field. +//! let blue = Bounded::::new::<0x18>(); +//! +//! // Setters can be chained. Values ranges are checked at compile-time. +//! let color = Rgb::zeroed() +//! // Compile-time bounds check of constant value. +//! .with_const_red::<0x10>() +//! .with_const_green::<0x1f>() +//! // A `Bounded` can also be passed. +//! .with_blue(blue); +//! +//! assert_eq!(color.red(), 0x10); +//! assert_eq!(color.green(), 0x1f); +//! assert_eq!(color.blue(), 0x18); +//! assert_eq!( +//! color.into_raw(), +//! (0x18 << Rgb::BLUE_SHIFT) + (0x1f << Rgb::GREEN_SHIFT) + 0x10, +//! ); +//! +//! // Convert to/from the backing storage type. +//! let raw: u16 = color.into(); +//! assert_eq!(Rgb::from(raw), color); +//! ``` +//! +//! # Syntax +//! +//! ```text +//! bitfield! { +//! #[attributes] +//! // Documentation for `Name`. +//! pub struct Name(storage_type) { +//! // `field_1` documentation. +//! hi:lo field_1; +//! // `field_2` documentation. +//! hi:lo field_2 => ConvertedType; +//! // `field_3` documentation. +//! hi:lo field_3 ?=> ConvertedType; +//! ... +//! } +//! } +//! ``` +//! +//! - `storage_type`: The underlying unsigned integer type ([`u8`], [`u16`], [`u32`], [`u64`]). +//! Signed integer storage types are not supported. +//! - `hi:lo`: Bit range (inclusive), where `hi >= lo`. +//! - `=> Type`: Optional infallible conversion (see [below](#infallible-conversion-)). +//! - `?=> Type`: Optional fallible conversion (see [below](#fallible-conversion-)). +//! - Documentation strings and attributes are optional. +//! +//! # Generated code +//! +//! Each field is internally represented as a [`Bounded`] parameterized by its bit width. Field +//! values can either be set/retrieved directly, or converted from/to another type. +//! +//! The use of [`Bounded`] for each field enforces bounds-checking (at build time or runtime) of +//! every value assigned to a field. This ensures that data is never accidentally truncated. +//! +//! The macro generates the bitfield type, [`From`] and [`Into`] implementations for its storage +//! type, as well as [`Debug`] and [`Zeroable`](pin_init::Zeroable) implementations. +//! +//! For each field, it also generates: +//! +//! - `field()`: Getter method for the field value. +//! - `with_field(value)`: Infallible setter; the argument type must fit within the field's width. +//! - `with_const_field::()`: `const` setter; the value is validated at compile time. +//! Usually shorter to use than `with_field` for constant values as it doesn't require +//! constructing a [`Bounded`]. +//! - `try_with_field(value)`: Fallible setter. Returns an error if the value is out of range. +//! - `FIELD_MASK`, `FIELD_SHIFT`, `FIELD_RANGE`: Constants for manual bit manipulation. +//! +//! # Reserved names for field identifiers +//! +//! Field identifiers are used to generate methods and associated constants on the bitfield type. +//! For a field named `field`, the macro may generate methods named `field`, `with_field`, +//! `with_const_field`, `try_with_field`, `__field` and `__with_field`, as well as constants named +//! `FIELD_MASK`, `FIELD_SHIFT` and `FIELD_RANGE`. +//! +//! Therefore, field identifiers must not use names that would collide with generated items for +//! any field in the same bitfield. The following prefixes are thus reserved for field identifiers: +//! +//! - `with_` +//! - `const_` +//! - `try_with_` +//! - `__` +//! +//! The field identifiers `from_raw`, `into_raw`, and `into` are also reserved. +//! +//! In addition, field identifiers should follow Rust `snake_case` conventions, since the associated +//! constants are generated by uppercasing the field name. +//! +//! # Implicit conversions +//! +//! Types that fit entirely within a field's bit width can be used directly with setters. For +//! example, [`bool`] works with single-bit fields, and [`u8`] works with 8-bit fields: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Flags(u32) { +//! 15:8 byte_field; +//! 0:0 flag; +//! } +//! } +//! +//! let flags = Flags::zeroed() +//! .with_byte_field(0x42_u8) +//! .with_flag(true); +//! +//! assert_eq!(flags.into_raw(), (0x42 << Flags::BYTE_FIELD_SHIFT) | 1); +//! ``` +//! +//! # Runtime bounds checking +//! +//! When a value is not known at compile time, use `try_with_field()` to check bounds at runtime: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Config(u8) { +//! 3:0 nibble; +//! } +//! } +//! +//! fn set_nibble(config: Config, value: u8) -> Result { +//! // Returns `EOVERFLOW` if `value > 0xf`. +//! config.try_with_nibble(value) +//! } +//! # Ok::<(), Error>(()) +//! ``` +//! +//! # Type conversion +//! +//! Fields can be automatically converted to/from a custom type using `=>` (infallible) or `?=>` +//! (fallible). The custom type must implement the appropriate [`From`] or [`TryFrom`] traits with +//! [`Bounded`]. +//! +//! ## Infallible conversion (`=>`) +//! +//! Use this when all possible bit patterns of a field map to valid values: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Power { +//! Off, +//! On, +//! } +//! +//! impl From> for Power { +//! fn from(v: Bounded) -> Self { +//! match *v { +//! 0 => Power::Off, +//! _ => Power::On, +//! } +//! } +//! } +//! +//! impl From for Bounded { +//! fn from(p: Power) -> Self { +//! (p as u32 != 0).into() +//! } +//! } +//! +//! bitfield! { +//! pub struct Control(u32) { +//! 0:0 power => Power; +//! } +//! } +//! +//! let ctrl = Control::zeroed().with_power(Power::On); +//! assert_eq!(ctrl.power(), Power::On); +//! ``` +//! +//! ## Fallible conversion (`?=>`) +//! +//! Use this when some bit patterns of a field are invalid. The getter returns a [`Result`]: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Mode { +//! Low = 0, +//! High = 1, +//! Auto = 2, +//! // 3 is invalid +//! } +//! +//! impl TryFrom> for Mode { +//! type Error = u32; +//! +//! fn try_from(v: Bounded) -> Result { +//! match *v { +//! 0 => Ok(Mode::Low), +//! 1 => Ok(Mode::High), +//! 2 => Ok(Mode::Auto), +//! n => Err(n), +//! } +//! } +//! } +//! +//! impl From for Bounded { +//! fn from(m: Mode) -> Self { +//! match m { +//! Mode::Low => Bounded::::new::<0>(), +//! Mode::High => Bounded::::new::<1>(), +//! Mode::Auto => Bounded::::new::<2>(), +//! } +//! } +//! } +//! +//! bitfield! { +//! pub struct Config(u32) { +//! 1:0 mode ?=> Mode; +//! } +//! } +//! +//! let cfg = Config::zeroed().with_mode(Mode::Auto); +//! assert_eq!(cfg.mode(), Ok(Mode::Auto)); +//! +//! // Invalid bit pattern returns an error. +//! assert_eq!(Config::from(0b11).mode(), Err(3)); +//! ``` +//! +//! # Bits outside of declared fields +//! +//! Bits of the storage type that are not part of any declared field are preserved by the setter +//! methods, and can only be modified through `from_raw` or the [`From`] implementation from the +//! storage type. +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Sparse(u8) { +//! 7:6 high; +//! // Bits 5:1 are not covered by any field. +//! 0:0 low; +//! } +//! } +//! +//! // Set the gap bits via `from_raw`, then mutate the declared fields. +//! let val = Sparse::from_raw(0b0010_1010) +//! .with_const_high::<0b11>() +//! .with_low(true); +//! +//! // Bits 5:1 are unchanged. +//! assert_eq!(val.into_raw(), 0b1110_1011); +//! ``` +//! +//! # Signed field values +//! +//! Bitfield storage types are unsigned. Since field getter methods return a [`Bounded`] of the +//! storage type, fields are also unsigned by default. +//! +//! If a field needs to encode a signed value, use a custom conversion type with `=>` or `?=>` to +//! perform the sign interpretation explicitly. +//! +//! [`Bounded`]: kernel::num::Bounded + +/// Defines a bitfield struct with bounds-checked accessors for individual bit ranges. +/// +/// See the [`mod@kernel::bitfield`] module for full documentation and examples. +#[macro_export] +macro_rules! bitfield { + // Entry point defining the bitfield struct, its implementations and its field accessors. + ( + $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!(@core + #[allow(non_camel_case_types)] + $(#[$attr])* $vis $name $storage + ); + $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); + }; + + // All rules below are helpers. + + // Defines the wrapper `$name` type and its conversions from/to the storage type. + (@core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { + $(#[$attr])* + #[repr(transparent)] + #[derive(Clone, Copy, PartialEq, Eq)] + $vis struct $name { + inner: $storage, + } + + #[allow(dead_code)] + impl $name { + /// Creates a bitfield from a raw value. + #[inline(always)] + $vis const fn from_raw(value: $storage) -> Self { + Self{ inner: value } + } + + /// Turns this bitfield into its raw value. + /// + /// This is similar to the [`From`] implementation, but is shorter to invoke in + /// most cases. + #[inline(always)] + $vis const fn into_raw(self) -> $storage { + self.inner + } + } + + // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. + unsafe impl ::pin_init::Zeroable for $name {} + + impl ::core::convert::From<$name> for $storage { + #[inline(always)] + fn from(val: $name) -> $storage { + val.into_raw() + } + } + + impl ::core::convert::From<$storage> for $name { + #[inline(always)] + fn from(val: $storage) -> $name { + Self::from_raw(val) + } + } + }; + + // Definitions requiring knowledge of individual fields: private and public field accessors, + // and `Debug` implementation. + (@fields $vis:vis $name:ident $storage:ty { + $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident + $(?=> $try_into_type:ty)? + $(=> $into_type:ty)? + ; + )* + } + ) => { + #[allow(dead_code)] + impl $name { + $( + $crate::bitfield!(@private_field_accessors $vis $name $storage : $hi:$lo $field); + $crate::bitfield!( + @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field + $(?=> $try_into_type)? + $(=> $into_type)? + ); + )* + } + + $crate::bitfield!(@debug $name { $($field;)* }); + }; + + // Private field accessors working with the exact `Bounded` type for the field. + ( + @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive = $lo..=$hi; + $vis const [<$field:upper _MASK>]: $storage = + ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); + $vis const [<$field:upper _SHIFT>]: u32 = $lo; + ); + + ::kernel::macros::paste!( + #[inline(always)] + fn [<__ $field>](self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { + // Left shift to align the field's MSB with the storage MSB. + const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); + // Right shift to move the top-aligned field to bit 0 of the storage. + const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; + + // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized + // output type. + let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( + self.inner << ALIGN_TOP + ); + val.shr::() + } + + #[inline(always)] + const fn [<__with_ $field>]( + mut self, + value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, + ) -> Self + { + const MASK: $storage = <$name>::[<$field:upper _MASK>]; + const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; + + let value = value.get() << SHIFT; + self.inner = (self.inner & !MASK) | value; + + self + } + ); + }; + + // Public accessors for fields infallibly (`=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:literal:$lo:literal $field:ident => $into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> $into_type + { + self.[<__ $field>]().into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields fallibly (`?=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::core::result::Result< + $try_into_type, + <$try_into_type as ::core::convert::TryFrom< + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + >>::Error + > + { + self.[<__ $field>]().try_into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $try_into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields not converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + { + self.[<__ $field>]() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the compile-time constant `VALUE`."] + #[inline(always)] + $vis const fn [](self) -> Self { + self.[<__with_ $field>]( + ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::() + ) + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> Self + where T: ::core::convert::Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, + { + self.[<__with_ $field>](value.into()) + } + + $(#[doc = $doc])* + #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> ::kernel::error::Result + where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, + { + Ok( + self.[<__with_ $field>]( + value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? + ) + ) + } + + ); + }; + + // `Debug` implementation. + (@debug $name:ident { $($field:ident;)* }) => { + impl ::kernel::fmt::Debug for $name { + #[inline] + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { + f.debug_struct(stringify!($name)) + .field("", &::kernel::prelude::fmt!("{:#x}", self.inner)) + $( + .field(stringify!($field), &self.$field()) + )* + .finish() + } + } + }; +} + +#[cfg(CONFIG_RUST_BITFIELD_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(rust_kernel_bitfield)] +mod tests { + use core::convert::TryFrom; + + use pin_init::Zeroable; + + use kernel::num::Bounded; + + // Enum types for testing `=>` and `?=>` conversions. + + #[derive(Debug, Clone, Copy, PartialEq)] + enum MemoryType { + Unmapped = 0, + Normal = 1, + Device = 2, + Reserved = 3, + } + + impl TryFrom> for MemoryType { + type Error = u64; + fn try_from(value: Bounded) -> Result { + match value.get() { + 0 => Ok(MemoryType::Unmapped), + 1 => Ok(MemoryType::Normal), + 2 => Ok(MemoryType::Device), + 3 => Ok(MemoryType::Reserved), + _ => Err(value.get()), + } + } + } + + impl From for Bounded { + #[inline(always)] + fn from(mt: MemoryType) -> Bounded { + Bounded::from_expr(mt as u64) + } + } + + #[derive(Debug, Clone, Copy, PartialEq)] + enum Priority { + Low = 0, + Medium = 1, + High = 2, + Critical = 3, + } + + impl From> for Priority { + fn from(value: Bounded) -> Self { + match value & 0x3 { + 0 => Priority::Low, + 1 => Priority::Medium, + 2 => Priority::High, + _ => Priority::Critical, + } + } + } + + impl From for Bounded { + #[inline(always)] + fn from(p: Priority) -> Bounded { + Bounded::from_expr(p as u16) + } + } + + bitfield! { + struct TestU64(u64) { + 63:63 field_63; + 61:52 field_61_52; + 51:16 field_51_16; + 15:12 field_15_12 ?=> MemoryType; + 11:9 field_11_9; + 1:1 field_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU16(u16) { + 15:8 field_15_8; + 7:4 field_7_4; // Partial overlap with `field_5_4`. + 5:4 field_5_4 => Priority; + 3:1 field_3_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU8(u8) { + 7:0 field_7_0; // Full byte overlap. + 7:4 field_7_4; + 3:2 field_3_2; + 1:1 field_1; + 0:0 field_0; + } + } + + // Single and multi-bit fields basic access. + #[test] + fn test_basic_access() { + // `TestU64`. + let mut val = TestU64::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + val = val.with_field_1(false); + assert!(!val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_11_9::<0x5>(); + assert_eq!(val.field_11_9(), 0x5); + assert_eq!(val.into_raw(), 0xA01); + + val = val.with_const_field_51_16::<0x123456>(); + assert_eq!(val.field_51_16(), 0x123456); + assert_eq!(val.into_raw(), 0x0012_3456_0A01); + + const MAX_FIELD_51_16: u64 = ::kernel::bits::genmask_u64(0..=35); + val = val.with_const_field_51_16::<{ MAX_FIELD_51_16 }>(); + assert_eq!(val.field_51_16(), MAX_FIELD_51_16); + + val = val.with_const_field_61_52::<0x3FF>(); + assert_eq!(val.field_61_52(), 0x3FF); + + val = val.with_field_63(true); + assert!(val.field_63().into_bool()); + + // `TestU16`. + let mut val = TestU16::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_3_1::<0x5>(); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.into_raw(), 0xB); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAB); + + val = val.with_const_field_15_8::<0x42>(); + assert_eq!(val.field_15_8(), 0x42); + assert_eq!(val.into_raw(), 0x42AB); + + // `TestU8`. + let mut val = TestU8::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x3); + + val = val.with_const_field_3_2::<0x3>(); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.into_raw(), 0xF); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAF); + } + + // `=>` infallible conversion. + #[test] + fn test_infallible_conversion() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::Low); + assert_eq!(val.field_5_4(), Priority::Low); + assert_eq!(val.into_raw() & 0x30, 0x00); + + val = val.with_field_5_4(Priority::Medium); + assert_eq!(val.field_5_4(), Priority::Medium); + assert_eq!(val.into_raw() & 0x30, 0x10); + + val = val.with_field_5_4(Priority::High); + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.into_raw() & 0x30, 0x20); + + val = val.with_field_5_4(Priority::Critical); + assert_eq!(val.field_5_4(), Priority::Critical); + assert_eq!(val.into_raw() & 0x30, 0x30); + } + + // `?=>` fallible conversion. + #[test] + fn test_fallible_conversion() { + let mut val = TestU64::zeroed(); + + val = val.with_field_15_12(MemoryType::Unmapped); + assert_eq!(val.field_15_12(), Ok(MemoryType::Unmapped)); + val = val.with_field_15_12(MemoryType::Normal); + assert_eq!(val.field_15_12(), Ok(MemoryType::Normal)); + val = val.with_field_15_12(MemoryType::Device); + assert_eq!(val.field_15_12(), Ok(MemoryType::Device)); + val = val.with_field_15_12(MemoryType::Reserved); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + + // `field_15_12` is 4 bits wide (0-15); `MemoryType` only covers 0-3, so 4-15 return `Err`. + let raw = (val.into_raw() & !::kernel::bits::genmask_u64(12..=15)) | (0x7 << 12); + assert_eq!(TestU64::from_raw(raw).field_15_12(), Err(0x7)); + } + + // Test that setting an overlapping field affects the overlapped one as expected. + #[test] + fn test_overlapping_fields() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::High); // High == 2 == 0b10. + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.field_7_4(), 0x2); // Bits 7:6 == 0, bits 5:4 == 0b10. + + val = val.with_const_field_7_4::<0xF>(); + assert_eq!(val.field_7_4(), 0xF); + assert_eq!(val.field_5_4(), Priority::Critical); // Bits 5:4 == 0b11. + + // `field_7_0` should encompass all other fields. + let mut val = TestU8::zeroed() + .with_field_0(true) + .with_field_1(true) + .with_const_field_3_2::<0x3>() + .with_const_field_7_4::<0xA>(); + assert_eq!(val.into_raw(), 0xAF); + + val = val.with_field_7_0(0x55); + assert_eq!(val.field_7_0(), 0x55); + assert!(val.field_0().into_bool()); + assert!(!val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x1); + assert_eq!(val.field_7_4(), 0x5); + } + + // Checks that bits not mapped to any field are left untouched. + #[test] + fn test_unallocated_bits() { + let gap_bits = (1u64 << 62) | 0x1FC; + + let set_all_fields = |val: TestU64| { + val.with_field_63(true) + .with_const_field_61_52::<0x155>() + .with_const_field_51_16::<0x123456>() + .with_field_15_12(MemoryType::Device) + .with_const_field_11_9::<0x5>() + .with_field_1(true) + .with_field_0(true) + }; + + // Gap bits to 0. + let val = set_all_fields(TestU64::from_raw(0)); + assert_eq!(val.into_raw() & gap_bits, 0); + + // Gap bits to 1. + let val = set_all_fields(TestU64::from_raw(gap_bits)); + assert_eq!(val.into_raw() & gap_bits, gap_bits); + } + + #[test] + fn test_try_with() { + let val = TestU64::zeroed().try_with_field_51_16(0x123456).unwrap(); + assert_eq!(val.field_51_16(), 0x123456); + + let err = TestU64::zeroed().try_with_field_51_16(u64::MAX); + assert_eq!(err, Err(::kernel::error::code::EOVERFLOW)); + + let val = TestU64::zeroed() + .try_with_field_51_16(0xABCDEF) + .and_then(|p| p.try_with_field_0(1)) + .unwrap(); + assert_eq!(val.field_51_16(), 0xABCDEF); + assert!(val.field_0().into_bool()); + } + + // `from_raw`/`into_raw` and `From`/`Into` round-trips. + #[test] + fn test_raw() { + let raw: u64 = 0xBFF0_0000_3123_3E03; + let val = TestU64::from_raw(raw); + assert_eq!(u64::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_11_9(), 0x7); + assert_eq!(val.field_51_16(), 0x3123); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + assert_eq!(val.field_61_52(), 0x3FF); + assert!(val.field_63().into_bool()); + + let raw: u16 = 0x42AB; + let val = TestU16::from_raw(raw); + assert_eq!(u16::from(val), raw); + assert!(val.field_0().into_bool()); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_15_8(), 0x42); + + let raw: u8 = 0xAF; + let val = TestU8::from_raw(raw); + assert_eq!(u8::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_7_0(), 0xAF); + } +} diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs new file mode 100644 index 000000000..b27e0ec80 --- /dev/null +++ b/rust/kernel/bitmap.rs @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! Rust API for bitmap. +//! +//! C headers: [`include/linux/bitmap.h`](srctree/include/linux/bitmap.h). + +use crate::alloc::{AllocError, Flags}; +use crate::bindings; +#[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] +use crate::pr_err; +use core::ptr::NonNull; + +/// Represents a C bitmap. Wraps underlying C bitmap API. +/// +/// # Invariants +/// +/// Must reference a `[c_ulong]` long enough to fit `data.len()` bits. +#[cfg_attr(CONFIG_64BIT, repr(align(8)))] +#[cfg_attr(not(CONFIG_64BIT), repr(align(4)))] +pub struct Bitmap { + data: [()], +} + +impl Bitmap { + /// Borrows a C bitmap. + /// + /// # Safety + /// + /// * `ptr` holds a non-null address of an initialized array of `unsigned long` + /// that is large enough to hold `nbits` bits. + /// * the array must not be freed for the lifetime of this [`Bitmap`] + /// * concurrent access only happens through atomic operations + pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap { + let data: *const [()] = core::ptr::slice_from_raw_parts(ptr.cast(), nbits); + // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // SAFETY: + // The caller guarantees that `data` (derived from `ptr` and `nbits`) + // points to a valid, initialized, and appropriately sized memory region + // that will not be freed for the lifetime 'a. + // We are casting `*const [()]` to `*const Bitmap`. The `Bitmap` + // struct is a ZST with a `data: [()]` field. This means its layout + // is compatible with a slice of `()`, and effectively it's a "thin pointer" + // (its size is 0 and alignment is 1). The `slice_from_raw_parts` + // function correctly encodes the length (number of bits, not elements) + // into the metadata of the fat pointer. Therefore, dereferencing this + // pointer as `&Bitmap` is safe given the caller's guarantees. + unsafe { &*(data as *const Bitmap) } + } + + /// Borrows a C bitmap exclusively. + /// + /// # Safety + /// + /// * `ptr` holds a non-null address of an initialized array of `unsigned long` + /// that is large enough to hold `nbits` bits. + /// * the array must not be freed for the lifetime of this [`Bitmap`] + /// * no concurrent access may happen. + pub unsafe fn from_raw_mut<'a>(ptr: *mut usize, nbits: usize) -> &'a mut Bitmap { + let data: *mut [()] = core::ptr::slice_from_raw_parts_mut(ptr.cast(), nbits); + // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // SAFETY: + // The caller guarantees that `data` (derived from `ptr` and `nbits`) + // points to a valid, initialized, and appropriately sized memory region + // that will not be freed for the lifetime 'a. + // Furthermore, the caller guarantees no concurrent access will happen, + // which upholds the exclusivity requirement for a mutable reference. + // Similar to `from_raw`, casting `*mut [()]` to `*mut Bitmap` is + // safe because `Bitmap` is a ZST with a `data: [()]` field, + // making its layout compatible with a slice of `()`. + unsafe { &mut *(data as *mut Bitmap) } + } + + /// Returns a raw pointer to the backing [`Bitmap`]. + pub fn as_ptr(&self) -> *const usize { + core::ptr::from_ref::(self).cast::() + } + + /// Returns a mutable raw pointer to the backing [`Bitmap`]. + pub fn as_mut_ptr(&mut self) -> *mut usize { + core::ptr::from_mut::(self).cast::() + } + + /// Returns length of this [`Bitmap`]. + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.data.len() + } +} + +/// Holds either a pointer to array of `unsigned long` or a small bitmap. +#[repr(C)] +union BitmapRepr { + bitmap: usize, + ptr: NonNull, +} + +macro_rules! bitmap_assert { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + } +} + +macro_rules! bitmap_assert_return { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + if !($cond) { + pr_err!($($arg)*); + return + } + } +} + +/// Represents an owned bitmap. +/// +/// Wraps underlying C bitmap API. See [`Bitmap`] for available +/// methods. +/// +/// # Examples +/// +/// Basic usage +/// +/// ``` +/// use kernel::alloc::flags::GFP_KERNEL; +/// use kernel::bitmap::BitmapVec; +/// +/// let mut b = BitmapVec::new(16, GFP_KERNEL)?; +/// +/// assert_eq!(16, b.len()); +/// for i in 0..16 { +/// if i % 4 == 0 { +/// b.set_bit(i); +/// } +/// } +/// assert_eq!(Some(0), b.next_bit(0)); +/// assert_eq!(Some(1), b.next_zero_bit(0)); +/// assert_eq!(Some(4), b.next_bit(1)); +/// assert_eq!(Some(5), b.next_zero_bit(4)); +/// assert_eq!(Some(12), b.last_bit()); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * `nbits` is `<= MAX_LEN`. +/// * if `nbits <= MAX_INLINE_LEN`, then `repr` is a `usize`. +/// * otherwise, `repr` holds a non-null pointer to an initialized +/// array of `unsigned long` that is large enough to hold `nbits` bits. +pub struct BitmapVec { + /// Representation of bitmap. + repr: BitmapRepr, + /// Length of this bitmap. Must be `<= MAX_LEN`. + nbits: usize, +} + +impl core::ops::Deref for BitmapVec { + type Target = Bitmap; + + fn deref(&self) -> &Bitmap { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { + // SAFETY: Bitmap is represented inline. + #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] + unsafe { + core::ptr::addr_of!(self.repr.bitmap) + } + } else { + // SAFETY: Bitmap is represented as array of `unsigned long`. + unsafe { self.repr.ptr.as_ptr() } + }; + + // SAFETY: We got the right pointer and invariants of [`Bitmap`] hold. + // An inline bitmap is treated like an array with single element. + unsafe { Bitmap::from_raw(ptr, self.nbits) } + } +} + +impl core::ops::DerefMut for BitmapVec { + fn deref_mut(&mut self) -> &mut Bitmap { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { + // SAFETY: Bitmap is represented inline. + #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] + unsafe { + core::ptr::addr_of_mut!(self.repr.bitmap) + } + } else { + // SAFETY: Bitmap is represented as array of `unsigned long`. + unsafe { self.repr.ptr.as_ptr() } + }; + + // SAFETY: We got the right pointer and invariants of [`BitmapVec`] hold. + // An inline bitmap is treated like an array with single element. + unsafe { Bitmap::from_raw_mut(ptr, self.nbits) } + } +} + +/// Enable ownership transfer to other threads. +/// +/// SAFETY: We own the underlying bitmap representation. +unsafe impl Send for BitmapVec {} + +/// Enable unsynchronized concurrent access to [`BitmapVec`] through shared references. +/// +/// SAFETY: `deref()` will return a reference to a [`Bitmap`]. Its methods +/// take immutable references are either atomic or read-only. +unsafe impl Sync for BitmapVec {} + +impl Drop for BitmapVec { + fn drop(&mut self) { + if self.nbits <= BitmapVec::MAX_INLINE_LEN { + return; + } + // SAFETY: `self.ptr` was returned by the C `bitmap_zalloc`. + // + // INVARIANT: there is no other use of the `self.ptr` after this + // call and the value is being dropped so the broken invariant is + // not observable on function exit. + unsafe { bindings::bitmap_free(self.repr.ptr.as_ptr()) }; + } +} + +impl BitmapVec { + /// The maximum possible length of a `BitmapVec`. + pub const MAX_LEN: usize = i32::MAX as usize; + + /// The maximum length that uses the inline representation. + pub const MAX_INLINE_LEN: usize = usize::BITS as usize; + + /// Construct a longest possible inline [`BitmapVec`]. + #[inline] + pub fn new_inline() -> Self { + // INVARIANT: `nbits <= MAX_INLINE_LEN`, so an inline bitmap is the right repr. + BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits: BitmapVec::MAX_INLINE_LEN, + } + } + + /// Constructs a new [`BitmapVec`]. + /// + /// Fails with [`AllocError`] when the [`BitmapVec`] could not be allocated. This + /// includes the case when `nbits` is greater than `MAX_LEN`. + #[inline] + pub fn new(nbits: usize, flags: Flags) -> Result { + if nbits <= BitmapVec::MAX_INLINE_LEN { + return Ok(BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits, + }); + } + if nbits > Self::MAX_LEN { + return Err(AllocError); + } + let nbits_u32 = u32::try_from(nbits).unwrap(); + // SAFETY: `MAX_INLINE_LEN < nbits` and `nbits <= MAX_LEN`. + let ptr = unsafe { bindings::bitmap_zalloc(nbits_u32, flags.as_raw()) }; + let ptr = NonNull::new(ptr).ok_or(AllocError)?; + // INVARIANT: `ptr` returned by C `bitmap_zalloc` and `nbits` checked. + Ok(BitmapVec { + repr: BitmapRepr { ptr }, + nbits, + }) + } + + /// Returns length of this [`Bitmap`]. + #[allow(clippy::len_without_is_empty)] + #[inline] + pub fn len(&self) -> usize { + self.nbits + } + + /// Fills this `Bitmap` with random bits. + #[cfg(CONFIG_FIND_BIT_BENCHMARK_RUST)] + pub fn fill_random(&mut self) { + // SAFETY: `self.as_mut_ptr` points to either an array of the + // appropriate length or one usize. + unsafe { + bindings::get_random_bytes( + self.as_mut_ptr().cast::(), + usize::div_ceil(self.nbits, bindings::BITS_PER_LONG as usize) + * bindings::BITS_PER_LONG as usize + / 8, + ); + } + } +} + +impl Bitmap { + /// Set bit with index `index`. + /// + /// ATTENTION: `set_bit` is non-atomic, which differs from the naming + /// convention in C code. The corresponding C function is `__set_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.nbits`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.nbits`. + #[inline] + pub fn set_bit(&mut self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: Bit `index` is within bounds. + unsafe { bindings::__set_bit(index, self.as_mut_ptr()) }; + } + + /// Set bit with index `index`, atomically. + /// + /// This is a relaxed atomic operation (no implied memory barriers). + /// + /// ATTENTION: The naming convention differs from C, where the corresponding + /// function is called `set_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn set_bit_atomic(&self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds and the caller has ensured that + // there is no mix of non-atomic and atomic operations. + unsafe { bindings::set_bit(index, self.as_ptr().cast_mut()) }; + } + + /// Clear `index` bit. + /// + /// ATTENTION: `clear_bit` is non-atomic, which differs from the naming + /// convention in C code. The corresponding C function is `__clear_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn clear_bit(&mut self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds. + unsafe { bindings::__clear_bit(index, self.as_mut_ptr()) }; + } + + /// Clear `index` bit, atomically. + /// + /// This is a relaxed atomic operation (no implied memory barriers). + /// + /// ATTENTION: The naming convention differs from C, where the corresponding + /// function is called `clear_bit`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than + /// or equal to `self.len()`, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than + /// or equal to `self.len()`. + #[inline] + pub fn clear_bit_atomic(&self, index: usize) { + bitmap_assert_return!( + index < self.len(), + "Bit `index` must be < {}, was {}", + self.len(), + index + ); + // SAFETY: `index` is within bounds and the caller has ensured that + // there is no mix of non-atomic and atomic operations. + unsafe { bindings::clear_bit(index, self.as_ptr().cast_mut()) }; + } + + /// Copy `src` into this [`Bitmap`] and set any remaining bits to zero. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; + /// use kernel::bitmap::BitmapVec; + /// + /// let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?; + /// + /// assert_eq!(None, long_bitmap.last_bit()); + /// + /// let mut short_bitmap = BitmapVec::new(16, GFP_KERNEL)?; + /// + /// short_bitmap.set_bit(7); + /// long_bitmap.copy_and_extend(&short_bitmap); + /// assert_eq!(Some(7), long_bitmap.last_bit()); + /// + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn copy_and_extend(&mut self, src: &Bitmap) { + let len = core::cmp::min(src.len(), self.len()); + // SAFETY: access to `self` and `src` is within bounds. + unsafe { + bindings::bitmap_copy_and_extend( + self.as_mut_ptr(), + src.as_ptr(), + len as u32, + self.len() as u32, + ) + }; + } + + /// Finds last set bit. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; + /// use kernel::bitmap::BitmapVec; + /// + /// let bitmap = BitmapVec::new(64, GFP_KERNEL)?; + /// + /// match bitmap.last_bit() { + /// Some(idx) => { + /// pr_info!("The last bit has index {idx}.\n"); + /// } + /// None => { + /// pr_info!("All bits in this bitmap are 0.\n"); + /// } + /// } + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn last_bit(&self) -> Option { + // SAFETY: `_find_next_bit` access is within bounds due to invariant. + let index = unsafe { bindings::_find_last_bit(self.as_ptr(), self.len()) }; + if index >= self.len() { + None + } else { + Some(index) + } + } + + /// Finds next set bit, starting from `start`. + /// + /// Returns `None` if `start` is greater or equal to `self.nbits`. + #[inline] + pub fn next_bit(&self, start: usize) -> Option { + bitmap_assert!( + start < self.len(), + "`start` must be < {} was {}", + self.len(), + start + ); + // SAFETY: `_find_next_bit` tolerates out-of-bounds arguments and returns a + // value larger than or equal to `self.len()` in that case. + let index = unsafe { bindings::_find_next_bit(self.as_ptr(), self.len(), start) }; + if index >= self.len() { + None + } else { + Some(index) + } + } + + /// Finds next zero bit, starting from `start`. + /// Returns `None` if `start` is greater than or equal to `self.len()`. + #[inline] + pub fn next_zero_bit(&self, start: usize) -> Option { + bitmap_assert!( + start < self.len(), + "`start` must be < {} was {}", + self.len(), + start + ); + // SAFETY: `_find_next_zero_bit` tolerates out-of-bounds arguments and returns a + // value larger than or equal to `self.len()` in that case. + let index = unsafe { bindings::_find_next_zero_bit(self.as_ptr(), self.len(), start) }; + if index >= self.len() { + None + } else { + Some(index) + } + } +} + +#[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bitmap)] +mod tests { + use super::*; + use kernel::alloc::flags::GFP_KERNEL; + + #[test] + fn bitmap_borrow() { + let fake_bitmap: [usize; 2] = [0, 0]; + let fake_bitmap_len = 2 * usize::BITS as usize; + // SAFETY: `fake_c_bitmap` is an array of expected length. + let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), fake_bitmap_len) }; + assert_eq!(fake_bitmap_len, b.len()); + assert_eq!(None, b.next_bit(0)); + } + + #[test] + fn bitmap_copy() { + let fake_bitmap: usize = 0xFF; + // SAFETY: `fake_c_bitmap` can be used as one-element array of expected length. + let b = unsafe { Bitmap::from_raw(core::ptr::addr_of!(fake_bitmap), 8) }; + assert_eq!(8, b.len()); + assert_eq!(None, b.next_zero_bit(0)); + } + + #[test] + fn bitmap_vec_new() -> Result<(), AllocError> { + let b = BitmapVec::new(0, GFP_KERNEL)?; + assert_eq!(0, b.len()); + + let b = BitmapVec::new(3, GFP_KERNEL)?; + assert_eq!(3, b.len()); + + let b = BitmapVec::new(1024, GFP_KERNEL)?; + assert_eq!(1024, b.len()); + + // Requesting too large values results in [`AllocError`]. + let res = BitmapVec::new(1 << 31, GFP_KERNEL); + assert!(res.is_err()); + Ok(()) + } + + #[test] + fn bitmap_set_clear_find() -> Result<(), AllocError> { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + // Zero-initialized + assert_eq!(None, b.next_bit(0)); + assert_eq!(Some(0), b.next_zero_bit(0)); + assert_eq!(None, b.last_bit()); + + b.set_bit(17); + + assert_eq!(Some(17), b.next_bit(0)); + assert_eq!(Some(17), b.next_bit(17)); + assert_eq!(None, b.next_bit(18)); + assert_eq!(Some(17), b.last_bit()); + + b.set_bit(107); + + assert_eq!(Some(17), b.next_bit(0)); + assert_eq!(Some(17), b.next_bit(17)); + assert_eq!(Some(107), b.next_bit(18)); + assert_eq!(Some(107), b.last_bit()); + + b.clear_bit(17); + + assert_eq!(Some(107), b.next_bit(0)); + assert_eq!(Some(107), b.last_bit()); + Ok(()) + } + + #[test] + fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { + // TODO: Kunit #[test]s do not support `cfg` yet, + // so we add it here in the body. + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + b.set_bit(2048); + b.set_bit_atomic(2048); + b.clear_bit(2048); + b.clear_bit_atomic(2048); + assert_eq!(None, b.next_bit(2048)); + assert_eq!(None, b.next_zero_bit(2048)); + assert_eq!(None, b.last_bit()); + } + Ok(()) + } + + // TODO: uncomment once kunit supports [should_panic] and `cfg`. + // #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + // #[test] + // #[should_panic] + // fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { + // let mut b = BitmapVec::new(128, GFP_KERNEL)?; + // + // b.set_bit(2048); + // } + + #[test] + fn bitmap_copy_and_extend() -> Result<(), AllocError> { + let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?; + + long_bitmap.set_bit(3); + long_bitmap.set_bit(200); + + let mut short_bitmap = BitmapVec::new(32, GFP_KERNEL)?; + + short_bitmap.set_bit(17); + + long_bitmap.copy_and_extend(&short_bitmap); + + // Previous bits have been cleared. + assert_eq!(Some(17), long_bitmap.next_bit(0)); + assert_eq!(Some(17), long_bitmap.last_bit()); + Ok(()) + } +} diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs new file mode 100644 index 000000000..2daead125 --- /dev/null +++ b/rust/kernel/bits.rs @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bit manipulation macros. +//! +//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h) + +use crate::prelude::*; +use core::ops::RangeInclusive; +use macros::paste; + +macro_rules! impl_bit_fn { + ( + $ty:ty + ) => { + paste! { + /// Computes `1 << n` if `n` is in bounds, i.e.: if `n` is smaller than + /// the maximum number of bits supported by the type. + /// + /// Returns [`None`] otherwise. + #[inline] + pub fn [](n: u32) -> Option<$ty> { + (1 as $ty).checked_shl(n) + } + + /// Computes `1 << n` by performing a compile-time assertion that `n` is + /// in bounds. + /// + /// This version is the default and should be used if `n` is known at + /// compile time. + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub const fn [](n: u32) -> $ty { + build_assert!(n < <$ty>::BITS); + (1 as $ty) << n + } + } + }; +} + +impl_bit_fn!(u64); +impl_bit_fn!(u32); +impl_bit_fn!(u16); +impl_bit_fn!(u8); + +macro_rules! impl_genmask_fn { + ( + $ty:ty, + $(#[$genmask_checked_ex:meta])*, + $(#[$genmask_ex:meta])* + ) => { + paste! { + /// Creates a contiguous bitmask for the given range by validating + /// the range at runtime. + /// + /// Returns [`None`] if the range is invalid, i.e.: if the start is + /// greater than the end or if the range is outside of the + /// representable range for the type. + $(#[$genmask_checked_ex])* + #[inline] + pub fn [](range: RangeInclusive) -> Option<$ty> { + let start = *range.start(); + let end = *range.end(); + + if start > end { + return None; + } + + let high = [](end)?; + let low = [](start)?; + Some((high | (high - 1)) & !(low - 1)) + } + + /// Creates a compile-time contiguous bitmask for the given range by + /// performing a compile-time assertion that the range is valid. + /// + /// This version is the default and should be used if the range is known + /// at compile time. + $(#[$genmask_ex])* + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub const fn [](range: RangeInclusive) -> $ty { + let start = *range.start(); + let end = *range.end(); + + build_assert!(start <= end); + + let high = [](end); + let low = [](start); + (high | (high - 1)) & !(low - 1) + } + } + }; +} + +impl_genmask_fn!( + u64, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u64; + /// assert_eq!(genmask_checked_u64(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u64(0..=63), Some(u64::MAX)); + /// assert_eq!(genmask_checked_u64(21..=39), Some(0x0000_00ff_ffe0_0000)); + /// + /// // `80` is out of the supported bit range. + /// assert_eq!(genmask_checked_u64(21..=80), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u64(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u64; + /// assert_eq!(genmask_u64(21..=39), 0x0000_00ff_ffe0_0000); + /// assert_eq!(genmask_u64(0..=0), 0b1); + /// assert_eq!(genmask_u64(0..=63), u64::MAX); + /// ``` +); + +impl_genmask_fn!( + u32, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u32; + /// assert_eq!(genmask_checked_u32(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u32(0..=31), Some(u32::MAX)); + /// assert_eq!(genmask_checked_u32(21..=31), Some(0xffe0_0000)); + /// + /// // `40` is out of the supported bit range. + /// assert_eq!(genmask_checked_u32(21..=40), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u32(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u32; + /// assert_eq!(genmask_u32(21..=31), 0xffe0_0000); + /// assert_eq!(genmask_u32(0..=0), 0b1); + /// assert_eq!(genmask_u32(0..=31), u32::MAX); + /// ``` +); + +impl_genmask_fn!( + u16, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u16; + /// assert_eq!(genmask_checked_u16(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u16(0..=15), Some(u16::MAX)); + /// assert_eq!(genmask_checked_u16(6..=15), Some(0xffc0)); + /// + /// // `20` is out of the supported bit range. + /// assert_eq!(genmask_checked_u16(6..=20), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u16(10..=5), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u16; + /// assert_eq!(genmask_u16(6..=15), 0xffc0); + /// assert_eq!(genmask_u16(0..=0), 0b1); + /// assert_eq!(genmask_u16(0..=15), u16::MAX); + /// ``` +); + +impl_genmask_fn!( + u8, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u8; + /// assert_eq!(genmask_checked_u8(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u8(0..=7), Some(u8::MAX)); + /// assert_eq!(genmask_checked_u8(6..=7), Some(0xc0)); + /// + /// // `10` is out of the supported bit range. + /// assert_eq!(genmask_checked_u8(6..=10), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u8(5..=2), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u8; + /// assert_eq!(genmask_u8(6..=7), 0xc0); + /// assert_eq!(genmask_u8(0..=0), 0b1); + /// assert_eq!(genmask_u8(0..=7), u8::MAX); + /// ``` +); diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs new file mode 100644 index 000000000..32c8d865a --- /dev/null +++ b/rust/kernel/block.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the block layer. + +pub mod mq; + +/// Bit mask for masking out [`SECTOR_SIZE`]. +pub const SECTOR_MASK: u32 = bindings::SECTOR_MASK; + +/// Sectors are size `1 << SECTOR_SHIFT`. +pub const SECTOR_SHIFT: u32 = bindings::SECTOR_SHIFT; + +/// Size of a sector. +pub const SECTOR_SIZE: u32 = bindings::SECTOR_SIZE; + +/// The difference between the size of a page and the size of a sector, +/// expressed as a power of two. +pub const PAGE_SECTORS_SHIFT: u32 = bindings::PAGE_SECTORS_SHIFT; diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs new file mode 100644 index 000000000..1fd0d54dd --- /dev/null +++ b/rust/kernel/block/mq.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides types for implementing block drivers that interface the +//! blk-mq subsystem. +//! +//! To implement a block device driver, a Rust module must do the following: +//! +//! - Implement [`Operations`] for a type `T`. +//! - Create a [`TagSet`]. +//! - Create a [`GenDisk`], via the [`GenDiskBuilder`]. +//! - Add the disk to the system by calling [`GenDiskBuilder::build`] passing in +//! the `TagSet` reference. +//! +//! The types available in this module that have direct C counterparts are: +//! +//! - The [`TagSet`] type that abstracts the C type `struct tag_set`. +//! - The [`GenDisk`] type that abstracts the C type `struct gendisk`. +//! - The [`Request`] type that abstracts the C type `struct request`. +//! +//! The kernel will interface with the block device driver by calling the method +//! implementations of the `Operations` trait. +//! +//! IO requests are passed to the driver as [`kernel::sync::aref::ARef`] +//! instances. The `Request` type is a wrapper around the C `struct request`. +//! The driver must mark end of processing by calling one of the +//! `Request::end`, methods. Failure to do so can lead to deadlock or timeout +//! errors. Please note that the C function `blk_mq_start_request` is implicitly +//! called when the request is queued with the driver. +//! +//! The `TagSet` is responsible for creating and maintaining a mapping between +//! `Request`s and integer ids as well as carrying a pointer to the vtable +//! generated by `Operations`. This mapping is useful for associating +//! completions from hardware with the correct `Request` instance. The `TagSet` +//! determines the maximum queue depth by setting the number of `Request` +//! instances available to the driver, and it determines the number of queues to +//! instantiate for the driver. If possible, a driver should allocate one queue +//! per core, to keep queue data local to a core. +//! +//! One `TagSet` instance can be shared between multiple `GenDisk` instances. +//! This can be useful when implementing drivers where one piece of hardware +//! with one set of IO resources are represented to the user as multiple disks. +//! +//! One significant difference between block device drivers implemented with +//! these Rust abstractions and drivers implemented in C, is that the Rust +//! drivers have to own a reference count on the `Request` type when the IO is +//! in flight. This is to ensure that the C `struct request` instances backing +//! the Rust `Request` instances are live while the Rust driver holds a +//! reference to the `Request`. In addition, the conversion of an integer tag to +//! a `Request` via the `TagSet` would not be sound without this bookkeeping. +//! +//! [`GenDisk`]: gen_disk::GenDisk +//! [`GenDisk`]: gen_disk::GenDisk +//! [`GenDiskBuilder`]: gen_disk::GenDiskBuilder +//! [`GenDiskBuilder::build`]: gen_disk::GenDiskBuilder::build +//! +//! # Examples +//! +//! ```rust +//! use kernel::{ +//! alloc::flags, +//! block::mq::*, +//! new_mutex, +//! prelude::*, +//! sync::{aref::ARef, Arc, Mutex}, +//! }; +//! +//! struct MyBlkDevice; +//! +//! #[vtable] +//! impl Operations for MyBlkDevice { +//! type QueueData = (); +//! +//! fn queue_rq(_queue_data: (), rq: ARef>, _is_last: bool) -> Result { +//! Request::end_ok(rq); +//! Ok(()) +//! } +//! +//! fn commit_rqs(_queue_data: ()) {} +//! +//! fn complete(rq: ARef>) { +//! Request::end_ok(rq) +//! .map_err(|_e| kernel::error::code::EIO) +//! .expect("Fatal error - expected to be able to end request"); +//! } +//! } +//! +//! let tagset: Arc> = +//! Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?; +//! let mut disk = gen_disk::GenDiskBuilder::new() +//! .capacity_sectors(4096) +//! .build(fmt!("myblk"), tagset, ())?; +//! +//! # Ok::<(), kernel::error::Error>(()) +//! ``` + +pub mod gen_disk; +mod operations; +mod request; +mod tag_set; + +pub use operations::Operations; +pub use request::Request; +pub use tag_set::TagSet; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs new file mode 100644 index 000000000..fc97dd873 --- /dev/null +++ b/rust/kernel/block/mq/gen_disk.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic disk abstraction. +//! +//! C header: [`include/linux/blkdev.h`](srctree/include/linux/blkdev.h) +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{Operations, TagSet}, + error::{self, from_err_ptr, Result}, + fmt::{self, Write}, + prelude::*, + static_lock_class, + str::NullTerminatedFormatter, + sync::Arc, + types::{ForeignOwnable, ScopeGuard}, +}; + +/// A builder for [`GenDisk`]. +/// +/// Use this struct to configure and add new [`GenDisk`] to the VFS. +pub struct GenDiskBuilder { + rotational: bool, + logical_block_size: u32, + physical_block_size: u32, + capacity_sectors: u64, +} + +impl Default for GenDiskBuilder { + fn default() -> Self { + Self { + rotational: false, + logical_block_size: bindings::PAGE_SIZE as u32, + physical_block_size: bindings::PAGE_SIZE as u32, + capacity_sectors: 0, + } + } +} + +impl GenDiskBuilder { + /// Create a new instance. + pub fn new() -> Self { + Self::default() + } + + /// Set the rotational media attribute for the device to be built. + pub fn rotational(mut self, rotational: bool) -> Self { + self.rotational = rotational; + self + } + + /// Validate block size by verifying that it is between 512 and `PAGE_SIZE`, + /// and that it is a power of two. + pub fn validate_block_size(size: u32) -> Result { + if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() { + Err(error::code::EINVAL) + } else { + Ok(()) + } + } + + /// Set the logical block size of the device to be built. + /// + /// This method will check that block size is a power of two and between 512 + /// and 4096. If not, an error is returned and the block size is not set. + /// + /// This is the smallest unit the storage device can address. It is + /// typically 4096 bytes. + pub fn logical_block_size(mut self, block_size: u32) -> Result { + Self::validate_block_size(block_size)?; + self.logical_block_size = block_size; + Ok(self) + } + + /// Set the physical block size of the device to be built. + /// + /// This method will check that block size is a power of two and between 512 + /// and 4096. If not, an error is returned and the block size is not set. + /// + /// This is the smallest unit a physical storage device can write + /// atomically. It is usually the same as the logical block size but may be + /// bigger. One example is SATA drives with 4096 byte physical block size + /// that expose a 512 byte logical block size to the operating system. + pub fn physical_block_size(mut self, block_size: u32) -> Result { + Self::validate_block_size(block_size)?; + self.physical_block_size = block_size; + Ok(self) + } + + /// Set the capacity of the device to be built, in sectors (512 bytes). + pub fn capacity_sectors(mut self, capacity: u64) -> Self { + self.capacity_sectors = capacity; + self + } + + /// Build a new `GenDisk` and add it to the VFS. + pub fn build( + self, + name: fmt::Arguments<'_>, + tagset: Arc>, + queue_data: T::QueueData, + ) -> Result> { + let data = queue_data.into_foreign(); + let recover_data = ScopeGuard::new(|| { + // SAFETY: T::QueueData was created by the call to `into_foreign()` above + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + let mut lim: bindings::queue_limits = pin_init::zeroed(); + + lim.logical_block_size = self.logical_block_size; + lim.physical_block_size = self.physical_block_size; + if self.rotational { + lim.features = bindings::BLK_FEAT_ROTATIONAL; + } + + // SAFETY: `tagset.raw_tag_set()` points to a valid and initialized tag set + let gendisk = from_err_ptr(unsafe { + bindings::__blk_mq_alloc_disk( + tagset.raw_tag_set(), + &mut lim, + data, + static_lock_class!().as_ptr(), + ) + })?; + + const TABLE: bindings::block_device_operations = bindings::block_device_operations { + submit_bio: None, + open: None, + release: None, + ioctl: None, + compat_ioctl: None, + check_events: None, + unlock_native_capacity: None, + getgeo: None, + set_read_only: None, + swap_slot_free_notify: None, + report_zones: None, + devnode: None, + alternative_gpt_sector: None, + get_unique_id: None, + // TODO: Set to `THIS_MODULE`. + owner: core::ptr::null_mut(), + pr_ops: core::ptr::null_mut(), + free_disk: None, + poll_bio: None, + }; + + // SAFETY: `gendisk` is a valid pointer as we initialized it above + unsafe { (*gendisk).fops = &TABLE }; + + let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| { + // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and + // has not been added to the VFS on this cleanup path. + unsafe { bindings::put_disk(gendisk) }; + // SAFETY: `data` came from `into_foreign()` above and has not been + // converted back on this cleanup path. + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + // The failure guard now owns both pieces of cleanup; the early guard + // must not run on this path anymore. + recover_data.dismiss(); + + let mut writer = NullTerminatedFormatter::new( + // SAFETY: `gendisk` points to a valid and initialized instance. We + // have exclusive access, since the disk is not added to the VFS + // yet. + unsafe { &mut (*gendisk).disk_name }, + ) + .ok_or(EINVAL)?; + writer.write_fmt(name)?; + + // SAFETY: `gendisk` points to a valid and initialized instance of + // `struct gendisk`. `set_capacity` takes a lock to synchronize this + // operation, so we will not race. + unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) }; + + crate::error::to_result( + // SAFETY: `gendisk` points to a valid and initialized instance of + // `struct gendisk`. + unsafe { + bindings::device_add_disk(core::ptr::null_mut(), gendisk, core::ptr::null_mut()) + }, + )?; + + cleanup_failure.dismiss(); + + // INVARIANT: `gendisk` was initialized above. + // INVARIANT: `gendisk` was added to the VFS via `device_add_disk` above. + // INVARIANT: `gendisk.queue.queue_data` is set to `data` in the call to + // `__blk_mq_alloc_disk` above. + Ok(GenDisk { + _tagset: tagset, + gendisk, + }) + } +} + +/// A generic block device. +/// +/// # Invariants +/// +/// - `gendisk` must always point to an initialized and valid `struct gendisk`. +/// - `gendisk` was added to the VFS through a call to +/// `bindings::device_add_disk`. +/// - `self.gendisk.queue.queuedata` is initialized by a call to `ForeignOwnable::into_foreign`. +pub struct GenDisk { + _tagset: Arc>, + gendisk: *mut bindings::gendisk, +} + +// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a +// `TagSet` It is safe to send this to other threads as long as T is Send. +unsafe impl Send for GenDisk {} + +impl Drop for GenDisk { + fn drop(&mut self) { + // SAFETY: By type invariant of `Self`, `self.gendisk` points to a valid + // and initialized instance of `struct gendisk`, and, `queuedata` was + // initialized with the result of a call to + // `ForeignOwnable::into_foreign`. + let queue_data = unsafe { (*(*self.gendisk).queue).queuedata }; + + // SAFETY: By type invariant, `self.gendisk` points to a valid and + // initialized instance of `struct gendisk`, and it was previously added + // to the VFS. + unsafe { bindings::del_gendisk(self.gendisk) }; + + // SAFETY: By type invariant, `self.gendisk` was added to the VFS, so + // `put_disk()` must follow `del_gendisk()` to drop the final gendisk + // reference and trigger the remaining release path. + unsafe { bindings::put_disk(self.gendisk) }; + + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with + // a call to `ForeignOwnable::into_foreign` to create `queuedata`. + // `ForeignOwnable::from_foreign` is only called here. + drop(unsafe { T::QueueData::from_foreign(queue_data) }); + } +} diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs new file mode 100644 index 000000000..861903e18 --- /dev/null +++ b/rust/kernel/block/mq/operations.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides an interface for blk-mq drivers to implement. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{request::RequestDataWrapper, Request}, + error::{from_result, Result}, + prelude::*, + sync::{aref::ARef, Refcount}, + types::ForeignOwnable, +}; +use core::marker::PhantomData; + +type ForeignBorrowed<'a, T> = ::Borrowed<'a>; + +/// Implement this trait to interface blk-mq as block devices. +/// +/// To implement a block device driver, implement this trait as described in the +/// [module level documentation]. The kernel will use the implementation of the +/// functions defined in this trait to interface a block device driver. Note: +/// There is no need for an exit_request() implementation, because the `drop` +/// implementation of the [`Request`] type will be invoked by automatically by +/// the C/Rust glue logic. +/// +/// [module level documentation]: kernel::block::mq +#[macros::vtable] +pub trait Operations: Sized { + /// Data associated with the `struct request_queue` that is allocated for + /// the `GenDisk` associated with this `Operations` implementation. + type QueueData: ForeignOwnable; + + /// Called by the kernel to queue a request with the driver. If `is_last` is + /// `false`, the driver is allowed to defer committing the request. + fn queue_rq( + queue_data: ForeignBorrowed<'_, Self::QueueData>, + rq: ARef>, + is_last: bool, + ) -> Result; + + /// Called by the kernel to indicate that queued requests should be submitted. + fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>); + + /// Called by the kernel when the request is completed. + fn complete(rq: ARef>); + + /// Called by the kernel to poll the device for completed requests. Only + /// used for poll queues. + fn poll() -> bool { + build_error!(crate::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A vtable for blk-mq to interact with a block device driver. +/// +/// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern +/// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`. +/// +/// For general documentation of these methods, see the kernel source +/// documentation related to `struct blk_mq_operations` in +/// [`include/linux/blk-mq.h`]. +/// +/// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h +pub(crate) struct OperationsVTable(PhantomData); + +impl OperationsVTable { + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - The caller of this function must ensure that the pointee of `bd` is + /// valid for reads for the duration of this function. + /// - This function must be called for an initialized and live `hctx`. That + /// is, `Self::init_hctx_callback` was called and + /// `Self::exit_hctx_callback()` was not yet called. + /// - `(*bd).rq` must point to an initialized and live `bindings:request`. + /// That is, `Self::init_request_callback` was called but + /// `Self::exit_request_callback` was not yet called for the request. + /// - `(*bd).rq` must be owned by the driver. That is, the block layer must + /// promise to not access the request until the driver calls + /// `bindings::blk_mq_end_request` for the request. + unsafe extern "C" fn queue_rq_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + bd: *const bindings::blk_mq_queue_data, + ) -> bindings::blk_status_t { + // SAFETY: `bd.rq` is valid as required by the safety requirement for + // this function. + let request = unsafe { &*(*bd).rq.cast::>() }; + + // One refcount for the ARef, one for being in flight + request.wrapper_ref().refcount().set(2); + + // SAFETY: + // - We own a refcount that we took above. We pass that to `ARef`. + // - By the safety requirements of this function, `request` is a valid + // `struct request` and the private data is properly initialized. + // - `rq` will be alive until `blk_mq_end_request` is called and is + // reference counted by `ARef` until then. + let rq = unsafe { Request::aref_from_raw((*bd).rq) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with + // a call to `ForeignOwnable::into_foreign` to create `queuedata`. + // `ForeignOwnable::from_foreign` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + + // SAFETY: We have exclusive access and we just set the refcount above. + unsafe { Request::start_unchecked(&rq) }; + + let ret = T::queue_rq( + queue_data, + rq, + // SAFETY: `bd` is valid as required by the safety requirement for + // this function. + unsafe { (*bd).last }, + ); + + if let Err(e) = ret { + e.to_blk_status() + } else { + bindings::BLK_STS_OK as bindings::blk_status_t + } + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. The caller + /// must ensure that `hctx` is valid. + unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) { + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a + // call to `ForeignOwnable::into_foreign()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + T::commit_rqs(queue_data) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. `rq` must + /// point to a valid request that has been marked as completed. The pointee + /// of `rq` must be valid for write for the duration of this function. + unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { + // SAFETY: This function can only be dispatched through + // `Request::complete`. We leaked a refcount then which we pick back up + // now. + let aref = unsafe { Request::aref_from_raw(rq) }; + T::complete(aref); + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. + unsafe extern "C" fn poll_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _iob: *mut bindings::io_comp_batch, + ) -> crate::ffi::c_int { + T::poll().into() + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. This + /// function may only be called once before `exit_hctx_callback` is called + /// for the same context. + unsafe extern "C" fn init_hctx_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _tagset_data: *mut crate::ffi::c_void, + _hctx_idx: crate::ffi::c_uint, + ) -> crate::ffi::c_int { + from_result(|| Ok(0)) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// This function may only be called by blk-mq C infrastructure. + unsafe extern "C" fn exit_hctx_callback( + _hctx: *mut bindings::blk_mq_hw_ctx, + _hctx_idx: crate::ffi::c_uint, + ) { + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - This function may only be called by blk-mq C infrastructure. + /// - `_set` must point to an initialized `TagSet`. + /// - `rq` must point to an initialized `bindings::request`. + /// - The allocation pointed to by `rq` must be at the size of `Request` + /// plus the size of `RequestDataWrapper`. + unsafe extern "C" fn init_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: crate::ffi::c_uint, + _numa_node: crate::ffi::c_int, + ) -> crate::ffi::c_int { + from_result(|| { + // SAFETY: By the safety requirements of this function, `rq` points + // to a valid allocation. + let pdu = unsafe { Request::wrapper_ptr(rq.cast::>()) }; + + // SAFETY: The refcount field is allocated but not initialized, so + // it is valid for writes. + unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; + + Ok(0) + }) + } + + /// This function is called by the C kernel. A pointer to this function is + /// installed in the `blk_mq_ops` vtable for the driver. + /// + /// # Safety + /// + /// - This function may only be called by blk-mq C infrastructure. + /// - `_set` must point to an initialized `TagSet`. + /// - `rq` must point to an initialized and valid `Request`. + unsafe extern "C" fn exit_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: crate::ffi::c_uint, + ) { + // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory + // for the request data. + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::(); + + // SAFETY: `pdu` is valid for read and write and is properly initialised. + unsafe { core::ptr::drop_in_place(pdu) }; + } + + const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops { + queue_rq: Some(Self::queue_rq_callback), + queue_rqs: None, + commit_rqs: Some(Self::commit_rqs_callback), + get_budget: None, + put_budget: None, + set_rq_budget_token: None, + get_rq_budget_token: None, + timeout: None, + poll: if T::HAS_POLL { + Some(Self::poll_callback) + } else { + None + }, + complete: Some(Self::complete_callback), + init_hctx: Some(Self::init_hctx_callback), + exit_hctx: Some(Self::exit_hctx_callback), + init_request: Some(Self::init_request_callback), + exit_request: Some(Self::exit_request_callback), + cleanup_rq: None, + busy: None, + map_queues: None, + #[cfg(CONFIG_BLK_DEBUG_FS)] + show_rq: None, + }; + + pub(crate) const fn build() -> &'static bindings::blk_mq_ops { + &Self::VTABLE + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs new file mode 100644 index 000000000..ce3e30c81 --- /dev/null +++ b/rust/kernel/block/mq/request.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides a wrapper for the C `struct request` type. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::Operations, + error::Result, + sync::{ + aref::{ARef, AlwaysRefCounted}, + atomic::Relaxed, + Refcount, + }, + types::Opaque, +}; +use core::{marker::PhantomData, ptr::NonNull}; + +/// A wrapper around a blk-mq [`struct request`]. This represents an IO request. +/// +/// # Implementation details +/// +/// There are four states for a request that the Rust bindings care about: +/// +/// 1. Request is owned by block layer (refcount 0). +/// 2. Request is owned by driver but with zero [`ARef`]s in existence +/// (refcount 1). +/// 3. Request is owned by driver with exactly one [`ARef`] in existence +/// (refcount 2). +/// 4. Request is owned by driver with more than one [`ARef`] in existence +/// (refcount > 2). +/// +/// +/// We need to track 1 and 2 to ensure we fail tag to request conversions for +/// requests that are not owned by the driver. +/// +/// We need to track 3 and 4 to ensure that it is safe to end the request and hand +/// back ownership to the block layer. +/// +/// Note that the driver can still obtain new `ARef` even if there is no `ARef`s in existence by +/// using `tag_to_rq`, hence the need to distinguish B and C. +/// +/// The states are tracked through the private `refcount` field of +/// `RequestDataWrapper`. This structure lives in the private data area of the C +/// [`struct request`]. +/// +/// # Invariants +/// +/// * `self.0` is a valid [`struct request`] created by the C portion of the +/// kernel. +/// * The private data area associated with this request must be an initialized +/// and valid `RequestDataWrapper`. +/// * `self` is reference counted by atomic modification of +/// `self.wrapper_ref().refcount()`. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h +/// +#[repr(transparent)] +pub struct Request(Opaque, PhantomData); + +impl Request { + /// Create an [`ARef`] from a [`struct request`] pointer. + /// + /// # Safety + /// + /// * The caller must own a refcount on `ptr` that is transferred to the + /// returned [`ARef`]. + /// * The type invariants for [`Request`] must hold for the pointee of `ptr`. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h + pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef { + // INVARIANT: By the safety requirements of this function, invariants are upheld. + // SAFETY: By the safety requirement of this function, we own a + // reference count that we can pass to `ARef`. + unsafe { ARef::from_raw(NonNull::new_unchecked(ptr.cast())) } + } + + /// Notify the block layer that a request is going to be processed now. + /// + /// The block layer uses this hook to do proper initializations such as + /// starting the timeout timer. It is a requirement that block device + /// drivers call this function when starting to process a request. + /// + /// # Safety + /// + /// The caller must have exclusive ownership of `self`, that is + /// `self.wrapper_ref().refcount() == 2`. + pub(crate) unsafe fn start_unchecked(this: &ARef) { + // SAFETY: By type invariant, `self.0` is a valid `struct request` and + // we have exclusive access. + unsafe { bindings::blk_mq_start_request(this.0.get()) }; + } + + /// Try to take exclusive ownership of `this` by dropping the refcount to 0. + /// This fails if `this` is not the only [`ARef`] pointing to the underlying + /// [`Request`]. + /// + /// If the operation is successful, [`Ok`] is returned with a pointer to the + /// C [`struct request`]. If the operation fails, `this` is returned in the + /// [`Err`] variant. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h + fn try_set_end(this: ARef) -> Result<*mut bindings::request, ARef> { + // To hand back the ownership, we need the current refcount to be 2. + // Since we can race with `TagSet::tag_to_rq`, this needs to atomically reduce + // refcount to 0. `Refcount` does not provide a way to do this, so use the underlying + // atomics directly. + if let Err(_old) = this + .wrapper_ref() + .refcount() + .as_atomic() + .cmpxchg(2, 0, Relaxed) + { + return Err(this); + } + + let request_ptr = this.0.get(); + core::mem::forget(this); + + Ok(request_ptr) + } + + /// Notify the block layer that the request has been completed without errors. + /// + /// This function will return [`Err`] if `this` is not the only [`ARef`] + /// referencing the request. + pub fn end_ok(this: ARef) -> Result<(), ARef> { + let request_ptr = Self::try_set_end(this)?; + + // SAFETY: By type invariant, `this.0` was a valid `struct request`. The + // success of the call to `try_set_end` guarantees that there are no + // `ARef`s pointing to this request. Therefore it is safe to hand it + // back to the block layer. + unsafe { + bindings::blk_mq_end_request( + request_ptr, + bindings::BLK_STS_OK as bindings::blk_status_t, + ) + }; + + Ok(()) + } + + /// Complete the request by scheduling `Operations::complete` for + /// execution. + /// + /// The function may be scheduled locally, via SoftIRQ or remotely via IPMI. + /// See `blk_mq_complete_request_remote` in [`blk-mq.c`] for details. + /// + /// [`blk-mq.c`]: srctree/block/blk-mq.c + pub fn complete(this: ARef) { + let ptr = ARef::into_raw(this).cast::().as_ptr(); + // SAFETY: By type invariant, `self.0` is a valid `struct request` + if !unsafe { bindings::blk_mq_complete_request_remote(ptr) } { + // SAFETY: We released a refcount above that we can reclaim here. + let this = unsafe { Request::aref_from_raw(ptr) }; + T::complete(this); + } + } + + /// Return a pointer to the [`RequestDataWrapper`] stored in the private area + /// of the request structure. + /// + /// # Safety + /// + /// - `this` must point to a valid allocation of size at least size of + /// [`Self`] plus size of [`RequestDataWrapper`]. + pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull { + let request_ptr = this.cast::(); + // SAFETY: By safety requirements for this function, `this` is a + // valid allocation. + let wrapper_ptr = + unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::() }; + // SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation + // and is not null. + unsafe { NonNull::new_unchecked(wrapper_ptr) } + } + + /// Return a reference to the [`RequestDataWrapper`] stored in the private + /// area of the request structure. + pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { + // SAFETY: By type invariant, `self.0` is a valid allocation. Further, + // the private data associated with this request is initialized and + // valid. The existence of `&self` guarantees that the private data is + // valid as a shared reference. + unsafe { Self::wrapper_ptr(core::ptr::from_ref(self).cast_mut()).as_ref() } + } +} + +/// A wrapper around data stored in the private area of the C [`struct request`]. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h +pub(crate) struct RequestDataWrapper { + /// The Rust request refcount has the following states: + /// + /// - 0: The request is owned by C block layer. + /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. + /// - 2+: There are [`ARef`] references to the request. + refcount: Refcount, +} + +impl RequestDataWrapper { + /// Return a reference to the refcount of the request that is embedding + /// `self`. + pub(crate) fn refcount(&self) -> &Refcount { + &self.refcount + } + + /// Return a pointer to the refcount of the request that is embedding the + /// pointee of `this`. + /// + /// # Safety + /// + /// - `this` must point to a live allocation of at least the size of `Self`. + pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { + // SAFETY: Because of the safety requirements of this function, the + // field projection is safe. + unsafe { &raw mut (*this).refcount } + } +} + +// SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut +// self` methods and `&self` methods that mutate `self` are internally +// synchronized. +unsafe impl Send for Request {} + +// SAFETY: Shared access is thread-safe for `Request`. `&self` methods that +// mutate `self` are internally synchronized` +unsafe impl Sync for Request {} + +// SAFETY: All instances of `Request` are reference counted. This +// implementation of `AlwaysRefCounted` ensure that increments to the ref count +// keeps the object alive in memory at least until a matching reference count +// decrement is executed. +unsafe impl AlwaysRefCounted for Request { + fn inc_ref(&self) { + self.wrapper_ref().refcount().inc(); + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: The type invariants of `ARef` guarantee that `obj` is valid + // for read. + let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() }; + // SAFETY: The type invariant of `Request` guarantees that the private + // data area is initialized and valid. + let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) }; + + #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))] + let is_zero = refcount.dec_and_test(); + + #[cfg(CONFIG_DEBUG_MISC)] + if is_zero { + panic!("Request reached refcount zero in Rust abstractions"); + } + } +} diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs new file mode 100644 index 000000000..dae9df408 --- /dev/null +++ b/rust/kernel/block/mq/tag_set.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides the `TagSet` struct to wrap the C `struct blk_mq_tag_set`. +//! +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) + +use core::pin::Pin; + +use crate::{ + bindings, + block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, + error::{self, Result}, + prelude::try_pin_init, + types::Opaque, +}; +use core::{convert::TryInto, marker::PhantomData}; +use pin_init::{pin_data, pinned_drop, PinInit}; + +/// A wrapper for the C `struct blk_mq_tag_set`. +/// +/// `struct blk_mq_tag_set` contains a `struct list_head` and so must be pinned. +/// +/// # Invariants +/// +/// - `inner` is initialized and valid. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct TagSet { + #[pin] + inner: Opaque, + _p: PhantomData, +} + +impl TagSet { + /// Try to create a new tag set + pub fn new( + nr_hw_queues: u32, + num_tags: u32, + num_maps: u32, + ) -> impl PinInit { + let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed(); + let tag_set: Result<_> = core::mem::size_of::() + .try_into() + .map(|cmd_size| { + bindings::blk_mq_tag_set { + ops: OperationsVTable::::build(), + nr_hw_queues, + timeout: 0, // 0 means default which is 30Hz in C + numa_node: bindings::NUMA_NO_NODE, + queue_depth: num_tags, + cmd_size, + flags: 0, + driver_data: core::ptr::null_mut::(), + nr_maps: num_maps, + ..tag_set + } + }) + .map(Opaque::new) + .map_err(|e| e.into()); + + try_pin_init!(TagSet { + inner <- tag_set.pin_chain(|tag_set| { + // SAFETY: we do not move out of `tag_set`. + let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) }; + // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`. + error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) + }), + _p: PhantomData, + }) + } + + /// Return the pointer to the wrapped `struct blk_mq_tag_set` + pub(crate) fn raw_tag_set(&self) -> *mut bindings::blk_mq_tag_set { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for TagSet { + fn drop(self: Pin<&mut Self>) { + // SAFETY: By type invariant `inner` is valid and has been properly + // initialized during construction. + unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) }; + } +} diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs new file mode 100644 index 000000000..3566f0234 --- /dev/null +++ b/rust/kernel/bug.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024, 2025 FUJITA Tomonori + +//! Support for BUG and WARN functionality. +//! +//! C header: [`include/asm-generic/bug.h`](srctree/include/asm-generic/bug.h) + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(CONFIG_DEBUG_BUGVERBOSE)] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + const _FILE: &[u8] = $file.as_bytes(); + // Plus one for null-terminator. + static FILE: [u8; _FILE.len() + 1] = { + let mut bytes = [0; _FILE.len() + 1]; + let mut i = 0; + while i < _FILE.len() { + bytes[i] = _FILE[i]; + i += 1; + } + bytes + }; + + // SAFETY: + // - `file`, `line`, `flags`, and `size` are all compile-time constants or + // symbols, preventing any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + file = sym FILE, + line = const line!(), + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + + if false { + _ = $file; + } + + // SAFETY: + // - `flags` and `size` are all compile-time constants, preventing + // any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, CONFIG_UML))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + + // SAFETY: It is always safe to call `warn_slowpath_fmt()` + // with a valid null-terminated string. + unsafe { + $crate::bindings::warn_slowpath_fmt( + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), + line!() as $crate::ffi::c_int, + $flags as $crate::ffi::c_uint, + ::core::ptr::null(), + ); + } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + + // SAFETY: It is always safe to call `WARN_ON()`. + unsafe { $crate::bindings::WARN_ON(true) } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(any(testlib, not(CONFIG_BUG)))] +macro_rules! warn_flags { + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; +} + +#[doc(hidden)] +pub const fn bugflag_taint(value: u32) -> u32 { + value << 8 +} + +/// Report a warning if `cond` is true and return the condition's evaluation result. +#[macro_export] +macro_rules! warn_on { + ($cond:expr) => {{ + let cond = $cond; + + #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] + const COND_STR: &str = file!(); + + if cond { + const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); + + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); + } + cond + }}; +} diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs new file mode 100644 index 000000000..c3acb9b68 --- /dev/null +++ b/rust/kernel/build_assert.rs @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Various assertions that happen during build-time. +//! +//! There are three types of build-time assertions that you can use: +//! - [`static_assert!`] +//! - [`const_assert!`] +//! - [`build_assert!`] +//! +//! The ones towards the bottom of the list are more expressive, while the ones towards the top of +//! the list are more robust and trigger earlier in the compilation pipeline. Therefore, you should +//! prefer the ones towards the top of the list wherever possible. +//! +//! # Choosing the correct assertion +//! +//! If you're asserting outside any bodies (e.g. initializers or function bodies), you should use +//! [`static_assert!`] as it is the only assertion that can be used in that context. +//! +//! Inside bodies, if your assertion condition does not depend on any variable or generics, you +//! should use [`static_assert!`]. If the condition depends on generics, but not variables +//! (including function arguments), you should use [`const_assert!`]. Otherwise, use +//! [`build_assert!`]. The same is true regardless if the function is `const fn`. +//! +//! ``` +//! // Outside any bodies. +//! static_assert!(core::mem::size_of::() == 1); +//! // `const_assert!` and `build_assert!` cannot be used here, they will fail to compile. +//! +//! #[inline(always)] +//! fn foo(v: usize) { +//! static_assert!(core::mem::size_of::() == 1); // Preferred. +//! const_assert!(core::mem::size_of::() == 1); // Discouraged. +//! build_assert!(core::mem::size_of::() == 1); // Discouraged. +//! +//! // `static_assert!(N > 1);` is not allowed. +//! const_assert!(N > 1); // Preferred. +//! build_assert!(N > 1); // Discouraged. +//! +//! // `static_assert!(v > 1);` is not allowed. +//! // `const_assert!(v > 1);` is not allowed. +//! build_assert!(v > 1); // Works. +//! } +//! ``` +//! +//! # Detailed behavior +//! +//! `static_assert!()` is equivalent to `static_assert` in C. It requires `expr` to be a constant +//! expression. This expression cannot refer to any generics. A `static_assert!(expr)` in a program +//! is always evaluated, regardless if the function it appears in is used or not. This is also the +//! only usable assertion outside a body. +//! +//! `const_assert!()` has no direct C equivalence. It is a more powerful version of +//! `static_assert!()`, where it may refer to generics in a function. Note that due to the ability +//! to refer to generics, the assertion is tied to a specific instance of a function. So if it is +//! used in a generic function that is not instantiated, the assertion will not be checked. For this +//! reason, `static_assert!()` is preferred wherever possible. +//! +//! `build_assert!()` is equivalent to `BUILD_BUG_ON`. It is even more powerful than +//! `const_assert!()` because it can be used to check tautologies that depend on runtime value (this +//! is the same as `BUILD_BUG_ON`). However, the assertion failure mechanism can possibly be +//! undefined symbols and linker errors, it is not developer friendly to debug, so it is recommended +//! to avoid it and prefer other two assertions where possible. + +#[doc(inline)] +pub use crate::{ + build_assert_macro as build_assert, + build_error, + const_assert, + static_assert, // +}; + +#[doc(hidden)] +pub use build_error::build_error as build_error_fn; + +/// Static assert (i.e. compile-time assert). +/// +/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`]. +/// +/// An optional panic message can be supplied after the expression. +/// Currently only a string literal without formatting is supported +/// due to constness limitations of the [`assert!`] macro. +/// +/// The feature may be added to Rust in the future: see [RFC 2790]. +/// +/// You cannot refer to generics or variables with [`static_assert!`]. If you need to refer to +/// generics, use [`const_assert!`]; if you need to refer to variables, use [`build_assert!`]. See +/// the [module documentation](self). +/// +/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert +/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert +/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790 +/// +/// # Examples +/// +/// ``` +/// static_assert!(42 > 24); +/// static_assert!(core::mem::size_of::() == 1); +/// +/// const X: &[u8] = b"bar"; +/// static_assert!(X[1] == b'a'); +/// +/// const fn f(x: i32) -> i32 { +/// x + 2 +/// } +/// static_assert!(f(40) == 42); +/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input."); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! static_assert { + ($condition:expr $(,$arg:literal)?) => { + const _: () = ::core::assert!($condition $(,$arg)?); + }; +} + +/// Assertion during constant evaluation. +/// +/// This is a more powerful version of [`static_assert!`] that can refer to generics inside +/// functions or implementation blocks. However, it also has a limitation where it can only appear +/// in places where statements can appear; for example, you cannot use it as an item in the module. +/// +/// [`static_assert!`] should be preferred if no generics are referred to in the condition. You +/// cannot refer to variables with [`const_assert!`] (even inside `const fn`); if you need the +/// capability, use [`build_assert!`]. See the [module documentation](self). +/// +/// # Examples +/// +/// ``` +/// fn foo() { +/// const_assert!(N > 1); +/// } +/// +/// fn bar() { +/// const_assert!(size_of::() > 0, "T cannot be ZST"); +/// } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! const_assert { + ($condition:expr $(,$arg:literal)?) => { + const { ::core::assert!($condition $(,$arg)?) }; + }; +} + +/// Fails the build if the code path calling `build_error!` can possibly be executed. +/// +/// If the macro is executed in const context, `build_error!` will panic. +/// If the compiler or optimizer cannot guarantee that `build_error!` can never +/// be called, a build error will be triggered. +/// +/// # Examples +/// +/// ``` +/// #[inline] +/// fn foo(a: usize) -> usize { +/// a.checked_add(1).unwrap_or_else(|| build_error!("overflow")) +/// } +/// +/// assert_eq!(foo(usize::MAX - 1), usize::MAX); // OK. +/// // foo(usize::MAX); // Fails to compile. +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! build_error { + () => {{ + $crate::build_assert::build_error_fn("") + }}; + ($msg:expr) => {{ + $crate::build_assert::build_error_fn($msg) + }}; +} + +/// Asserts that a boolean expression is `true` at compile time. +/// +/// If the condition is evaluated to `false` in const context, `build_assert!` +/// will panic. If the compiler or optimizer cannot guarantee the condition will +/// be evaluated to `true`, a build error will be triggered. +/// +/// When a condition depends on a function argument, the function must be annotated with +/// `#[inline(always)]`. Without this attribute, the compiler may choose to not inline the +/// function, preventing it from optimizing out the error path. +/// +/// If the assertion condition does not depend on any variables or generics, you should use +/// [`static_assert!`]. If the assertion condition does not depend on variables, but does depend on +/// generics, you should use [`const_assert!`]. See the [module documentation](self). +/// +/// # Examples +/// +/// ``` +/// #[inline(always)] // Important. +/// fn bar(n: usize) { +/// build_assert!(n > 1); +/// } +/// +/// fn foo() { +/// bar(2); +/// } +/// +/// #[inline(always)] // Important. +/// const fn const_bar(n: usize) { +/// build_assert!(n > 1); +/// } +/// +/// const _: () = const_bar(2); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! build_assert_macro { + ($cond:expr $(,)?) => {{ + if !$cond { + $crate::build_assert::build_error_fn(concat!("assertion failed: ", stringify!($cond))); + } + }}; + ($cond:expr, $msg:expr) => {{ + if !$cond { + $crate::build_assert::build_error_fn($msg); + } + }}; +} diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs new file mode 100644 index 000000000..7abbd0767 --- /dev/null +++ b/rust/kernel/clk.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Clock abstractions. +//! +//! C header: [`include/linux/clk.h`](srctree/include/linux/clk.h) +//! +//! Reference: + +use crate::ffi::c_ulong; + +/// The frequency unit. +/// +/// Represents a frequency in hertz, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::clk::Hertz; +/// +/// let hz = 1_000_000_000; +/// let rate = Hertz(hz); +/// +/// assert_eq!(rate.as_hz(), hz); +/// assert_eq!(rate, Hertz(hz)); +/// assert_eq!(rate, Hertz::from_khz(hz / 1_000)); +/// assert_eq!(rate, Hertz::from_mhz(hz / 1_000_000)); +/// assert_eq!(rate, Hertz::from_ghz(hz / 1_000_000_000)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct Hertz(pub c_ulong); + +impl Hertz { + const KHZ_TO_HZ: c_ulong = 1_000; + const MHZ_TO_HZ: c_ulong = 1_000_000; + const GHZ_TO_HZ: c_ulong = 1_000_000_000; + + /// Create a new instance from kilohertz (kHz) + pub const fn from_khz(khz: c_ulong) -> Self { + Self(khz * Self::KHZ_TO_HZ) + } + + /// Create a new instance from megahertz (MHz) + pub const fn from_mhz(mhz: c_ulong) -> Self { + Self(mhz * Self::MHZ_TO_HZ) + } + + /// Create a new instance from gigahertz (GHz) + pub const fn from_ghz(ghz: c_ulong) -> Self { + Self(ghz * Self::GHZ_TO_HZ) + } + + /// Get the frequency in hertz + pub const fn as_hz(&self) -> c_ulong { + self.0 + } + + /// Get the frequency in kilohertz + pub const fn as_khz(&self) -> c_ulong { + self.0 / Self::KHZ_TO_HZ + } + + /// Get the frequency in megahertz + pub const fn as_mhz(&self) -> c_ulong { + self.0 / Self::MHZ_TO_HZ + } + + /// Get the frequency in gigahertz + pub const fn as_ghz(&self) -> c_ulong { + self.0 / Self::GHZ_TO_HZ + } +} + +impl From for c_ulong { + fn from(freq: Hertz) -> Self { + freq.0 + } +} + +#[cfg(CONFIG_COMMON_CLK)] +mod common_clk { + use super::Hertz; + use crate::{ + device::Device, + error::{from_err_ptr, to_result, Result}, + prelude::*, + }; + + use core::{ops::Deref, ptr}; + + /// A reference-counted clock. + /// + /// Rust abstraction for the C [`struct clk`]. + /// + /// # Invariants + /// + /// A [`Clk`] instance holds either a pointer to a valid [`struct clk`] created by the C + /// portion of the kernel or a `NULL` pointer. + /// + /// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`Clk`]. + /// + /// # Examples + /// + /// The following example demonstrates how to obtain and configure a clock for a device. + /// + /// ``` + /// use kernel::clk::{Clk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = Clk::get(dev, Some(c"apb_clk"))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + #[repr(transparent)] + pub struct Clk(*mut bindings::clk); + + // SAFETY: It is safe to call `clk_put` on another thread than where `clk_get` was called. + unsafe impl Send for Clk {} + + // SAFETY: It is safe to call any combination of the `&self` methods in parallel, as the + // methods are synchronized internally. + unsafe impl Sync for Clk {} + + impl Clk { + /// Gets [`Clk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get`] API. + /// + /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); + + // SAFETY: It is safe to call [`clk_get`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`Clk`] goes out of scope. + Ok(Self(from_err_ptr(unsafe { + bindings::clk_get(dev.as_raw(), con_id) + })?)) + } + + /// Obtain the raw [`struct clk`] pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::clk { + self.0 + } + + /// Enable the clock. + /// + /// Equivalent to the kernel's [`clk_enable`] API. + /// + /// [`clk_enable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_enable + #[inline] + pub fn enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_enable`]. + to_result(unsafe { bindings::clk_enable(self.as_raw()) }) + } + + /// Disable the clock. + /// + /// Equivalent to the kernel's [`clk_disable`] API. + /// + /// [`clk_disable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_disable + #[inline] + pub fn disable(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable`]. + unsafe { bindings::clk_disable(self.as_raw()) }; + } + + /// Prepare the clock. + /// + /// Equivalent to the kernel's [`clk_prepare`] API. + /// + /// [`clk_prepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_prepare + #[inline] + pub fn prepare(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare`]. + to_result(unsafe { bindings::clk_prepare(self.as_raw()) }) + } + + /// Unprepare the clock. + /// + /// Equivalent to the kernel's [`clk_unprepare`] API. + /// + /// [`clk_unprepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_unprepare + #[inline] + pub fn unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_unprepare`]. + unsafe { bindings::clk_unprepare(self.as_raw()) }; + } + + /// Prepare and enable the clock. + /// + /// Equivalent to calling [`Clk::prepare`] followed by [`Clk::enable`]. + #[inline] + pub fn prepare_enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare_enable`]. + to_result(unsafe { bindings::clk_prepare_enable(self.as_raw()) }) + } + + /// Disable and unprepare the clock. + /// + /// Equivalent to calling [`Clk::disable`] followed by [`Clk::unprepare`]. + #[inline] + pub fn disable_unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable_unprepare`]. + unsafe { bindings::clk_disable_unprepare(self.as_raw()) }; + } + + /// Get clock's rate. + /// + /// Equivalent to the kernel's [`clk_get_rate`] API. + /// + /// [`clk_get_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_rate + #[inline] + pub fn rate(&self) -> Hertz { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_get_rate`]. + Hertz(unsafe { bindings::clk_get_rate(self.as_raw()) }) + } + + /// Set clock's rate. + /// + /// Equivalent to the kernel's [`clk_set_rate`] API. + /// + /// [`clk_set_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_set_rate + #[inline] + pub fn set_rate(&self, rate: Hertz) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_set_rate`]. + to_result(unsafe { bindings::clk_set_rate(self.as_raw(), rate.as_hz()) }) + } + } + + impl Drop for Clk { + fn drop(&mut self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for [`clk_put`]. + unsafe { bindings::clk_put(self.as_raw()) }; + } + } + + /// A reference-counted optional clock. + /// + /// A lightweight wrapper around an optional [`Clk`]. An [`OptionalClk`] represents a [`Clk`] + /// that a driver can function without but may improve performance or enable additional + /// features when available. + /// + /// # Invariants + /// + /// An [`OptionalClk`] instance encapsulates a [`Clk`] with either a valid [`struct clk`] or + /// `NULL` pointer. + /// + /// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`OptionalClk`]. + /// + /// # Examples + /// + /// The following example demonstrates how to obtain and configure an optional clock for a + /// device. The code functions correctly whether or not the clock is available. + /// + /// ``` + /// use kernel::clk::{OptionalClk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = OptionalClk::get(dev, Some(c"apb_clk"))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + pub struct OptionalClk(Clk); + + impl OptionalClk { + /// Gets [`OptionalClk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get_optional`] API. + /// + /// [`clk_get_optional`]: + /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); + + // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`OptionalClk`] goes out of + // scope. + Ok(Self(Clk(from_err_ptr(unsafe { + bindings::clk_get_optional(dev.as_raw(), con_id) + })?))) + } + } + + // Make [`OptionalClk`] behave like [`Clk`]. + impl Deref for OptionalClk { + type Target = Clk; + + fn deref(&self) -> &Clk { + &self.0 + } + } +} + +#[cfg(CONFIG_COMMON_CLK)] +pub use common_clk::*; diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs new file mode 100644 index 000000000..cd082b83e --- /dev/null +++ b/rust/kernel/configfs.rs @@ -0,0 +1,1047 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! configfs interface: Userspace-driven Kernel Object Configuration +//! +//! configfs is an in-memory pseudo file system for configuration of kernel +//! modules. Please see the [C documentation] for details and intended use of +//! configfs. +//! +//! This module does not support the following configfs features: +//! +//! - Items. All group children are groups. +//! - Symlink support. +//! - `disconnect_notify` hook. +//! - Default groups. +//! +//! See the [`rust_configfs.rs`] sample for a full example use of this module. +//! +//! C header: [`include/linux/configfs.h`](srctree/include/linux/configfs.h) +//! +//! # Examples +//! +//! ```ignore +//! use kernel::alloc::flags; +//! use kernel::configfs_attrs; +//! use kernel::configfs; +//! use kernel::new_mutex; +//! use kernel::page::PAGE_SIZE; +//! use kernel::sync::Mutex; +//! use kernel::ThisModule; +//! +//! #[pin_data] +//! struct RustConfigfs { +//! #[pin] +//! config: configfs::Subsystem, +//! } +//! +//! impl kernel::InPlaceModule for RustConfigfs { +//! fn init(_module: &'static ThisModule) -> impl PinInit { +//! pr_info!("Rust configfs sample (init)\n"); +//! +//! let item_type = configfs_attrs! { +//! container: configfs::Subsystem, +//! data: Configuration, +//! attributes: [ +//! message: 0, +//! bar: 1, +//! ], +//! }; +//! +//! try_pin_init!(Self { +//! config <- configfs::Subsystem::new( +//! c"rust_configfs", item_type, Configuration::new() +//! ), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct Configuration { +//! message: &'static CStr, +//! #[pin] +//! bar: Mutex<(KBox<[u8; PAGE_SIZE]>, usize)>, +//! } +//! +//! impl Configuration { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! message: c"Hello World\n", +//! bar <- new_mutex!((KBox::new([0; PAGE_SIZE], flags::GFP_KERNEL)?, 0)), +//! }) +//! } +//! } +//! +//! #[vtable] +//! impl configfs::AttributeOperations<0> for Configuration { +//! type Data = Configuration; +//! +//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result { +//! pr_info!("Show message\n"); +//! let data = container.message; +//! page[0..data.len()].copy_from_slice(data); +//! Ok(data.len()) +//! } +//! } +//! +//! #[vtable] +//! impl configfs::AttributeOperations<1> for Configuration { +//! type Data = Configuration; +//! +//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result { +//! pr_info!("Show bar\n"); +//! let guard = container.bar.lock(); +//! let data = guard.0.as_slice(); +//! let len = guard.1; +//! page[0..len].copy_from_slice(&data[0..len]); +//! Ok(len) +//! } +//! +//! fn store(container: &Configuration, page: &[u8]) -> Result { +//! pr_info!("Store bar\n"); +//! let mut guard = container.bar.lock(); +//! guard.0[0..page.len()].copy_from_slice(page); +//! guard.1 = page.len(); +//! Ok(()) +//! } +//! } +//! ``` +//! +//! [C documentation]: srctree/Documentation/filesystems/configfs.rst +//! [`rust_configfs.rs`]: srctree/samples/rust/rust_configfs.rs + +use crate::alloc::flags; +use crate::container_of; +use crate::page::PAGE_SIZE; +use crate::prelude::*; +use crate::str::CString; +use crate::sync::Arc; +use crate::sync::ArcBorrow; +use crate::types::Opaque; +use core::cell::UnsafeCell; +use core::marker::PhantomData; + +/// A configfs subsystem. +/// +/// This is the top level entrypoint for a configfs hierarchy. To register +/// with configfs, embed a field of this type into your kernel module struct. +#[pin_data(PinnedDrop)] +pub struct Subsystem { + #[pin] + subsystem: Opaque, + #[pin] + data: Data, +} + +// SAFETY: We do not provide any operations on `Subsystem`. +unsafe impl Sync for Subsystem {} + +// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads. +unsafe impl Send for Subsystem {} + +impl Subsystem { + /// Create an initializer for a [`Subsystem`]. + /// + /// The subsystem will appear in configfs as a directory name given by + /// `name`. The attributes available in directory are specified by + /// `item_type`. + pub fn new( + name: &'static CStr, + item_type: &'static ItemType, Data>, + data: impl PinInit, + ) -> impl PinInit { + try_pin_init!(Self { + subsystem <- pin_init::init_zeroed().chain( + |place: &mut Opaque| { + // SAFETY: We initialized the required fields of `place.group` above. + unsafe { + bindings::config_group_init_type_name( + &mut (*place.get()).su_group, + name.as_char_ptr(), + item_type.as_ptr(), + ) + }; + + // SAFETY: `place.su_mutex` is valid for use as a mutex. + unsafe { + bindings::__mutex_init( + &mut (*place.get()).su_mutex, + kernel::optional_name!().as_char_ptr(), + kernel::static_lock_class!().as_ptr(), + ) + } + Ok(()) + } + ), + data <- data, + }) + .pin_chain(|this| { + crate::error::to_result( + // SAFETY: We initialized `this.subsystem` according to C API contract above. + unsafe { bindings::configfs_register_subsystem(this.subsystem.get()) }, + ) + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Subsystem { + fn drop(self: Pin<&mut Self>) { + // SAFETY: We registered `self.subsystem` in the initializer returned by `Self::new`. + unsafe { bindings::configfs_unregister_subsystem(self.subsystem.get()) }; + // SAFETY: We initialized the mutex in `Subsystem::new`. + unsafe { bindings::mutex_destroy(&raw mut (*self.subsystem.get()).su_mutex) }; + } +} + +/// Trait that allows offset calculations for structs that embed a +/// `bindings::config_group`. +/// +/// Users of the configfs API should not need to implement this trait. +/// +/// # Safety +/// +/// - Implementers of this trait must embed a `bindings::config_group`. +/// - Methods must be implemented according to method documentation. +pub unsafe trait HasGroup { + /// Return the address of the `bindings::config_group` embedded in [`Self`]. + /// + /// # Safety + /// + /// - `this` must be a valid allocation of at least the size of [`Self`]. + unsafe fn group(this: *const Self) -> *const bindings::config_group; + + /// Return the address of the [`Self`] that `group` is embedded in. + /// + /// # Safety + /// + /// - `group` must point to the `bindings::config_group` that is embedded in + /// [`Self`]. + unsafe fn container_of(group: *const bindings::config_group) -> *const Self; +} + +// SAFETY: `Subsystem` embeds a field of type `bindings::config_group` +// within the `subsystem` field. +unsafe impl HasGroup for Subsystem { + unsafe fn group(this: *const Self) -> *const bindings::config_group { + // SAFETY: By impl and function safety requirement this projection is in bounds. + unsafe { &raw const (*(*this).subsystem.get()).su_group } + } + + unsafe fn container_of(group: *const bindings::config_group) -> *const Self { + // SAFETY: By impl and function safety requirement this projection is in bounds. + let c_subsys_ptr = unsafe { container_of!(group, bindings::configfs_subsystem, su_group) }; + let opaque_ptr = c_subsys_ptr.cast::>(); + // SAFETY: By impl and function safety requirement, `opaque_ptr` and the + // pointer it returns, are within the same allocation. + unsafe { container_of!(opaque_ptr, Subsystem, subsystem) } + } +} + +/// A configfs group. +/// +/// To add a subgroup to configfs, pass this type as `ctype` to +/// [`crate::configfs_attrs`] when creating a group in [`GroupOperations::make_group`]. +#[pin_data] +pub struct Group { + #[pin] + group: Opaque, + #[pin] + data: Data, +} + +impl Group { + /// Create an initializer for a new group. + /// + /// When instantiated, the group will appear as a directory with the name + /// given by `name` and it will contain attributes specified by `item_type`. + pub fn new( + name: CString, + item_type: &'static ItemType, Data>, + data: impl PinInit, + ) -> impl PinInit { + try_pin_init!(Self { + group <- pin_init::init_zeroed().chain(|v: &mut Opaque| { + let place = v.get(); + let name = name.to_bytes_with_nul().as_ptr(); + // SAFETY: It is safe to initialize a group once it has been zeroed. + unsafe { + bindings::config_group_init_type_name(place, name.cast(), item_type.as_ptr()) + }; + Ok(()) + }), + data <- data, + }) + } +} + +// SAFETY: `Group` embeds a field of type `bindings::config_group` +// within the `group` field. +unsafe impl HasGroup for Group { + unsafe fn group(this: *const Self) -> *const bindings::config_group { + Opaque::cast_into( + // SAFETY: By impl and function safety requirements this field + // projection is within bounds of the allocation. + unsafe { &raw const (*this).group }, + ) + } + + unsafe fn container_of(group: *const bindings::config_group) -> *const Self { + let opaque_ptr = group.cast::>(); + // SAFETY: By impl and function safety requirement, `opaque_ptr` and + // pointer it returns will be in the same allocation. + unsafe { container_of!(opaque_ptr, Self, group) } + } +} + +/// # Safety +/// +/// `this` must be a valid pointer. +/// +/// If `this` does not represent the root group of a configfs subsystem, +/// `this` must be a pointer to a `bindings::config_group` embedded in a +/// `Group`. +/// +/// Otherwise, `this` must be a pointer to a `bindings::config_group` that +/// is embedded in a `bindings::configfs_subsystem` that is embedded in a +/// `Subsystem`. +unsafe fn get_group_data<'a, Parent>(this: *mut bindings::config_group) -> &'a Parent { + // SAFETY: `this` is a valid pointer. + let is_root = unsafe { (*this).cg_subsys.is_null() }; + + if !is_root { + // SAFETY: By C API contact,`this` was returned from a call to + // `make_group`. The pointer is known to be embedded within a + // `Group`. + unsafe { &(*Group::::container_of(this)).data } + } else { + // SAFETY: By C API contract, `this` is a pointer to the + // `bindings::config_group` field within a `Subsystem`. + unsafe { &(*Subsystem::container_of(this)).data } + } +} + +struct GroupOperationsVTable(PhantomData<(Parent, Child)>); + +impl GroupOperationsVTable +where + Parent: GroupOperations, + Child: 'static, +{ + /// # Safety + /// + /// `this` must be a valid pointer. + /// + /// If `this` does not represent the root group of a configfs subsystem, + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// Otherwise, `this` must be a pointer to a `bindings::config_group` that + /// is embedded in a `bindings::configfs_subsystem` that is embedded in a + /// `Subsystem`. + /// + /// `name` must point to a null terminated string. + unsafe extern "C" fn make_group( + this: *mut bindings::config_group, + name: *const kernel::ffi::c_char, + ) -> *mut bindings::config_group { + // SAFETY: By function safety requirements of this function, this call + // is safe. + let parent_data = unsafe { get_group_data(this) }; + + let group_init = match Parent::make_group( + parent_data, + // SAFETY: By function safety requirements, name points to a null + // terminated string. + unsafe { CStr::from_char_ptr(name) }, + ) { + Ok(init) => init, + Err(e) => return e.to_ptr(), + }; + + let child_group = > as InPlaceInit>>::try_pin_init( + group_init, + flags::GFP_KERNEL, + ); + + match child_group { + Ok(child_group) => { + let child_group_ptr = child_group.into_raw(); + // SAFETY: We allocated the pointee of `child_ptr` above as a + // `Group`. + unsafe { Group::::group(child_group_ptr) }.cast_mut() + } + Err(e) => e.to_ptr(), + } + } + + /// # Safety + /// + /// If `this` does not represent the root group of a configfs subsystem, + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// Otherwise, `this` must be a pointer to a `bindings::config_group` that + /// is embedded in a `bindings::configfs_subsystem` that is embedded in a + /// `Subsystem`. + /// + /// `item` must point to a `bindings::config_item` within a + /// `bindings::config_group` within a `Group`. + unsafe extern "C" fn drop_item( + this: *mut bindings::config_group, + item: *mut bindings::config_item, + ) { + // SAFETY: By function safety requirements of this function, this call + // is safe. + let parent_data = unsafe { get_group_data(this) }; + + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + let c_child_group_ptr = unsafe { container_of!(item, bindings::config_group, cg_item) }; + // SAFETY: By function safety requirements, `c_child_group_ptr` is + // embedded within a `Group`. + let r_child_group_ptr = unsafe { Group::::container_of(c_child_group_ptr) }; + + if Parent::HAS_DROP_ITEM { + // SAFETY: We called `into_raw` to produce `r_child_group_ptr` in + // `make_group`. + let arc: Arc> = unsafe { Arc::from_raw(r_child_group_ptr.cast_mut()) }; + + Parent::drop_item(parent_data, arc.as_arc_borrow()); + arc.into_raw(); + } + + // SAFETY: By C API contract, we are required to drop a refcount on + // `item`. + unsafe { bindings::config_item_put(item) }; + } + + const VTABLE: bindings::configfs_group_operations = bindings::configfs_group_operations { + make_item: None, + make_group: Some(Self::make_group), + disconnect_notify: None, + drop_item: Some(Self::drop_item), + is_visible: None, + is_bin_visible: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_group_operations { + &Self::VTABLE + } +} + +struct ItemOperationsVTable(PhantomData<(Container, Data)>); + +impl ItemOperationsVTable, Data> +where + Data: 'static, +{ + /// # Safety + /// + /// `this` must be a pointer to a `bindings::config_group` embedded in a + /// `Group`. + /// + /// This function will destroy the pointee of `this`. The pointee of `this` + /// must not be accessed after the function returns. + unsafe extern "C" fn release(this: *mut bindings::config_item) { + // SAFETY: By function safety requirements, `this` is embedded in a + // `config_group`. + let c_group_ptr = unsafe { kernel::container_of!(this, bindings::config_group, cg_item) }; + // SAFETY: By function safety requirements, `c_group_ptr` is + // embedded within a `Group`. + let r_group_ptr = unsafe { Group::::container_of(c_group_ptr) }; + + // SAFETY: We called `into_raw` on `r_group_ptr` in + // `make_group`. + let pin_self: Arc> = unsafe { Arc::from_raw(r_group_ptr.cast_mut()) }; + drop(pin_self); + } + + const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations { + release: Some(Self::release), + allow_link: None, + drop_link: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_item_operations { + &Self::VTABLE + } +} + +impl ItemOperationsVTable, Data> { + const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations { + release: None, + allow_link: None, + drop_link: None, + }; + + const fn vtable_ptr() -> *const bindings::configfs_item_operations { + &Self::VTABLE + } +} + +/// Operations implemented by configfs groups that can create subgroups. +/// +/// Implement this trait on structs that embed a [`Subsystem`] or a [`Group`]. +#[vtable] +pub trait GroupOperations { + /// The child data object type. + /// + /// This group will create subgroups (subdirectories) backed by this kind of + /// object. + type Child: 'static; + + /// Creates a new subgroup. + /// + /// The kernel will call this method in response to `mkdir(2)` in the + /// directory representing `this`. + /// + /// To accept the request to create a group, implementations should + /// return an initializer of a `Group`. To prevent creation, + /// return a suitable error. + fn make_group(&self, name: &CStr) -> Result, Error>>; + + /// Prepares the group for removal from configfs. + /// + /// The kernel will call this method before the directory representing `_child` is removed from + /// configfs. + /// + /// Implementations can use this method to do house keeping before configfs drops its + /// reference to `Child`. + /// + /// NOTE: "drop" in the name of this function is not related to the Rust drop term. Rather, the + /// name is inherited from the callback name in the underlying C code. + fn drop_item(&self, _child: ArcBorrow<'_, Group>) { + kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A configfs attribute. +/// +/// An attribute appears as a file in configfs, inside a folder that represent +/// the group that the attribute belongs to. +#[repr(transparent)] +pub struct Attribute { + attribute: Opaque, + _p: PhantomData<(O, Data)>, +} + +// SAFETY: We do not provide any operations on `Attribute`. +unsafe impl Sync for Attribute {} + +// SAFETY: Ownership of `Attribute` can safely be transferred to other threads. +unsafe impl Send for Attribute {} + +impl Attribute +where + O: AttributeOperations, +{ + /// # Safety + /// + /// `item` must be embedded in a `bindings::config_group`. + /// + /// If `item` does not represent the root group of a configfs subsystem, + /// the group must be embedded in a `Group`. + /// + /// Otherwise, the group must be a embedded in a + /// `bindings::configfs_subsystem` that is embedded in a `Subsystem`. + /// + /// `page` must point to a writable buffer of size at least [`PAGE_SIZE`]. + unsafe extern "C" fn show( + item: *mut bindings::config_item, + page: *mut kernel::ffi::c_char, + ) -> isize { + let c_group: *mut bindings::config_group = + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + unsafe { container_of!(item, bindings::config_group, cg_item) }; + + // SAFETY: The function safety requirements for this function satisfy + // the conditions for this call. + let data: &Data = unsafe { get_group_data(c_group) }; + + // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`. + let ret = O::show(data, unsafe { &mut *(page.cast::<[u8; PAGE_SIZE]>()) }); + + match ret { + Ok(size) => size as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `item` must be embedded in a `bindings::config_group`. + /// + /// If `item` does not represent the root group of a configfs subsystem, + /// the group must be embedded in a `Group`. + /// + /// Otherwise, the group must be a embedded in a + /// `bindings::configfs_subsystem` that is embedded in a `Subsystem`. + /// + /// `page` must point to a readable buffer of size at least `size`. + unsafe extern "C" fn store( + item: *mut bindings::config_item, + page: *const kernel::ffi::c_char, + size: usize, + ) -> isize { + let c_group: *mut bindings::config_group = + // SAFETY: By function safety requirements, `item` is embedded in a + // `config_group`. + unsafe { container_of!(item, bindings::config_group, cg_item) }; + + // SAFETY: The function safety requirements for this function satisfy + // the conditions for this call. + let data: &Data = unsafe { get_group_data(c_group) }; + + let ret = O::store( + data, + // SAFETY: By function safety requirements, `page` is readable + // for at least `size`. + unsafe { core::slice::from_raw_parts(page.cast(), size) }, + ); + + match ret { + Ok(()) => size as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// Create a new attribute. + /// + /// The attribute will appear as a file with name given by `name`. + pub const fn new(name: &'static CStr) -> Self { + Self { + attribute: Opaque::new(bindings::configfs_attribute { + ca_name: crate::str::as_char_ptr_in_const_context(name), + ca_owner: core::ptr::null_mut(), + ca_mode: 0o660, + show: Some(Self::show), + store: if O::HAS_STORE { + Some(Self::store) + } else { + None + }, + }), + _p: PhantomData, + } + } +} + +/// Operations supported by an attribute. +/// +/// Implement this trait on type and pass that type as generic parameter when +/// creating an [`Attribute`]. The type carrying the implementation serve no +/// purpose other than specifying the attribute operations. +/// +/// This trait must be implemented on the `Data` type of for types that +/// implement `HasGroup`. The trait must be implemented once for each +/// attribute of the group. The constant type parameter `ID` maps the +/// implementation to a specific `Attribute`. `ID` must be passed when declaring +/// attributes via the [`kernel::configfs_attrs`] macro, to tie +/// `AttributeOperations` implementations to concrete named attributes. +#[vtable] +pub trait AttributeOperations { + /// The type of the object that contains the field that is backing the + /// attribute for this operation. + type Data; + + /// Renders the value of an attribute. + /// + /// This function is called by the kernel to read the value of an attribute. + /// + /// Implementations should write the rendering of the attribute to `page` + /// and return the number of bytes written. + fn show(data: &Self::Data, page: &mut [u8; PAGE_SIZE]) -> Result; + + /// Stores the value of an attribute. + /// + /// This function is called by the kernel to update the value of an attribute. + /// + /// Implementations should parse the value from `page` and update internal + /// state to reflect the parsed value. + fn store(_data: &Self::Data, _page: &[u8]) -> Result { + kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR) + } +} + +/// A list of attributes. +/// +/// This type is used to construct a new [`ItemType`]. It represents a list of +/// [`Attribute`] that will appear in the directory representing a [`Group`]. +/// Users should not directly instantiate this type, rather they should use the +/// [`kernel::configfs_attrs`] macro to declare a static set of attributes for a +/// group. +/// +/// # Note +/// +/// Instances of this type are constructed statically at compile by the +/// [`kernel::configfs_attrs`] macro. +#[repr(transparent)] +pub struct AttributeList( + /// Null terminated Array of pointers to [`Attribute`]. The type is [`c_void`] + /// to conform to the C API. + UnsafeCell<[*mut kernel::ffi::c_void; N]>, + PhantomData, +); + +// SAFETY: Ownership of `AttributeList` can safely be transferred to other threads. +unsafe impl Send for AttributeList {} + +// SAFETY: We do not provide any operations on `AttributeList` that need synchronization. +unsafe impl Sync for AttributeList {} + +impl AttributeList { + /// # Safety + /// + /// This function must only be called by the [`kernel::configfs_attrs`] + /// macro. + #[doc(hidden)] + pub const unsafe fn new() -> Self { + Self(UnsafeCell::new([core::ptr::null_mut(); N]), PhantomData) + } + + /// # Safety + /// + /// The caller must ensure that there are no other concurrent accesses to + /// `self`. That is, the caller has exclusive access to `self.` + #[doc(hidden)] + pub const unsafe fn add( + &'static self, + attribute: &'static Attribute, + ) where + O: AttributeOperations, + { + // We need a space at the end of our list for a null terminator. + const { assert!(I < N - 1, "Invalid attribute index") }; + + // SAFETY: By function safety requirements, we have exclusive access to + // `self` and the reference created below will be exclusive. + unsafe { (&mut *self.0.get())[I] = core::ptr::from_ref(attribute).cast_mut().cast() }; + } +} + +/// A representation of the attributes that will appear in a [`Group`] or +/// [`Subsystem`]. +/// +/// Users should not directly instantiate objects of this type. Rather, they +/// should use the [`kernel::configfs_attrs`] macro to statically declare the +/// shape of a [`Group`] or [`Subsystem`]. +#[pin_data] +pub struct ItemType { + #[pin] + item_type: Opaque, + _p: PhantomData<(Container, Data)>, +} + +// SAFETY: We do not provide any operations on `ItemType` that need synchronization. +unsafe impl Sync for ItemType {} + +// SAFETY: Ownership of `ItemType` can safely be transferred to other threads. +unsafe impl Send for ItemType {} + +macro_rules! impl_item_type { + ($tpe:ty) => { + impl ItemType<$tpe, Data> { + #[doc(hidden)] + pub const fn new_with_child_ctor( + owner: &'static ThisModule, + attributes: &'static AttributeList, + ) -> Self + where + Data: GroupOperations, + Child: 'static, + { + Self { + item_type: Opaque::new(bindings::config_item_type { + ct_owner: owner.as_ptr(), + ct_group_ops: GroupOperationsVTable::::vtable_ptr().cast_mut(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), + ct_bin_attrs: core::ptr::null_mut(), + }), + _p: PhantomData, + } + } + + #[doc(hidden)] + pub const fn new( + owner: &'static ThisModule, + attributes: &'static AttributeList, + ) -> Self { + Self { + item_type: Opaque::new(bindings::config_item_type { + ct_owner: owner.as_ptr(), + ct_group_ops: core::ptr::null_mut(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), + ct_bin_attrs: core::ptr::null_mut(), + }), + _p: PhantomData, + } + } + } + }; +} + +impl_item_type!(Subsystem); +impl_item_type!(Group); + +impl ItemType { + fn as_ptr(&self) -> *const bindings::config_item_type { + self.item_type.get() + } +} + +/// Define a list of configfs attributes statically. +/// +/// Invoking the macro in the following manner: +/// +/// ```ignore +/// let item_type = configfs_attrs! { +/// container: configfs::Subsystem, +/// data: Configuration, +/// child: Child, +/// attributes: [ +/// message: 0, +/// bar: 1, +/// ], +/// }; +/// ``` +/// +/// Expands the following output: +/// +/// ```ignore +/// let item_type = { +/// static CONFIGURATION_MESSAGE_ATTR: kernel::configfs::Attribute< +/// 0, +/// Configuration, +/// Configuration, +/// > = unsafe { +/// kernel::configfs::Attribute::new({ +/// const S: &str = "message\u{0}"; +/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul( +/// S.as_bytes() +/// ) { +/// Ok(v) => v, +/// Err(_) => { +/// core::panicking::panic_fmt(core::const_format_args!( +/// "string contains interior NUL" +/// )); +/// } +/// }; +/// C +/// }) +/// }; +/// +/// static CONFIGURATION_BAR_ATTR: kernel::configfs::Attribute< +/// 1, +/// Configuration, +/// Configuration +/// > = unsafe { +/// kernel::configfs::Attribute::new({ +/// const S: &str = "bar\u{0}"; +/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul( +/// S.as_bytes() +/// ) { +/// Ok(v) => v, +/// Err(_) => { +/// core::panicking::panic_fmt(core::const_format_args!( +/// "string contains interior NUL" +/// )); +/// } +/// }; +/// C +/// }) +/// }; +/// +/// const N: usize = (1usize + (1usize + 0usize)) + 1usize; +/// +/// static CONFIGURATION_ATTRS: kernel::configfs::AttributeList = +/// unsafe { kernel::configfs::AttributeList::new() }; +/// +/// { +/// const N: usize = 0usize; +/// unsafe { CONFIGURATION_ATTRS.add::(&CONFIGURATION_MESSAGE_ATTR) }; +/// } +/// +/// { +/// const N: usize = (1usize + 0usize); +/// unsafe { CONFIGURATION_ATTRS.add::(&CONFIGURATION_BAR_ATTR) }; +/// } +/// +/// static CONFIGURATION_TPE: +/// kernel::configfs::ItemType ,Configuration> +/// = kernel::configfs::ItemType::< +/// configfs::Subsystem, +/// Configuration +/// >::new_with_child_ctor::( +/// ::kernel::module::this_module::(), +/// &CONFIGURATION_ATTRS +/// ); +/// +/// &CONFIGURATION_TPE +/// } +/// ``` +#[allow(clippy::crate_in_macro_def)] +#[macro_export] +macro_rules! configfs_attrs { + ( + container: $container:ty, + data: $data:ty, + attributes: [ + $($name:ident: $attr:literal),* $(,)? + ] $(,)? + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child(), + @no_child(x), + @attrs($($name $attr)*), + @eat($($name $attr,)*), + @assign(), + @cnt(0usize), + ) + }; + ( + container: $container:ty, + data: $data:ty, + child: $child:ty, + attributes: [ + $($name:ident: $attr:literal),* $(,)? + ] $(,)? + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child($child), + @no_child(), + @attrs($($name $attr)*), + @eat($($name $attr,)*), + @assign(), + @cnt(0usize), + ) + }; + (count: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($aname:ident $aattr:literal)*), + @eat($name:ident $attr:literal, $($rname:ident $rattr:literal,)*), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => { + $crate::configfs_attrs!( + count: + @container($container), + @data($data), + @child($($child)?), + @no_child($($no_child)?), + @attrs($($aname $aattr)*), + @eat($($rname $rattr,)*), + @assign($($assign)* { + const N: usize = $cnt; + // The following macro text expands to a call to `Attribute::add`. + + // SAFETY: By design of this macro, the name of the variable we + // invoke the `add` method on below, is not visible outside of + // the macro expansion. The macro does not operate concurrently + // on this variable, and thus we have exclusive access to the + // variable. + unsafe { + $crate::macros::paste!( + [< $data:upper _ATTRS >] + .add::(&[< $data:upper _ $name:upper _ATTR >]) + ) + }; + }), + @cnt(1usize + $cnt), + ) + }; + (count: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($aname:ident $aattr:literal)*), + @eat(), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => + { + $crate::configfs_attrs!( + final: + @container($container), + @data($data), + @child($($child)?), + @no_child($($no_child)?), + @attrs($($aname $aattr)*), + @assign($($assign)*), + @cnt($cnt), + ) + }; + (final: + @container($container:ty), + @data($data:ty), + @child($($child:ty)?), + @no_child($($no_child:ident)?), + @attrs($($name:ident $attr:literal)*), + @assign($($assign:block)*), + @cnt($cnt:expr), + ) => + { + $crate::macros::paste!{ + { + $( + // SAFETY: We are expanding `configfs_attrs`. + static [< $data:upper _ $name:upper _ATTR >]: + $crate::configfs::Attribute<$attr, $data, $data> = + unsafe { + $crate::configfs::Attribute::new( + $crate::c_str!(::core::stringify!($name)), + ) + }; + )* + + + // We need space for a null terminator. + const N: usize = $cnt + 1usize; + + // SAFETY: We are expanding `configfs_attrs`. + static [< $data:upper _ATTRS >]: + $crate::configfs::AttributeList = + unsafe { $crate::configfs::AttributeList::new() }; + + $($assign)* + + $( + const [<$no_child:upper>]: bool = true; + + static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> = + $crate::configfs::ItemType::<$container, $data>::new::( + $crate::module::this_module::(), + &[<$ data:upper _ATTRS >] + ); + )? + + $( + static [< $data:upper _TPE >]: + $crate::configfs::ItemType<$container, $data> = + $crate::configfs::ItemType::<$container, $data>:: + new_with_child_ctor::( + $crate::module::this_module::(), + &[<$ data:upper _ATTRS >] + ); + )? + + & [< $data:upper _TPE >] + } + } + }; + +} + +pub use crate::configfs_attrs; diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs new file mode 100644 index 000000000..cb6c0338e --- /dev/null +++ b/rust/kernel/cpu.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic CPU definitions. +//! +//! C header: [`include/linux/cpu.h`](srctree/include/linux/cpu.h) + +use crate::{bindings, device::Device, error::Result, prelude::ENODEV}; + +/// Returns the maximum number of possible CPUs in the current system configuration. +#[inline] +pub fn nr_cpu_ids() -> u32 { + #[cfg(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS))] + { + bindings::NR_CPUS + } + + #[cfg(not(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS)))] + // SAFETY: `nr_cpu_ids` is a valid global provided by the kernel. + unsafe { + bindings::nr_cpu_ids + } +} + +/// The CPU ID. +/// +/// Represents a CPU identifier as a wrapper around an [`u32`]. +/// +/// # Invariants +/// +/// The CPU ID lies within the range `[0, nr_cpu_ids())`. +/// +/// # Examples +/// +/// ``` +/// use kernel::cpu::CpuId; +/// +/// let cpu = 0; +/// +/// // SAFETY: 0 is always a valid CPU number. +/// let id = unsafe { CpuId::from_u32_unchecked(cpu) }; +/// +/// assert_eq!(id.as_u32(), cpu); +/// assert!(CpuId::from_i32(0).is_some()); +/// assert!(CpuId::from_i32(-1).is_none()); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct CpuId(u32); + +impl CpuId { + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_i32_unchecked(id: i32) -> Self { + debug_assert!(id >= 0); + debug_assert!((id as u32) < nr_cpu_ids()); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id as u32) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_i32(id: i32) -> Option { + if id < 0 || id as u32 >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id as u32)) + } + } + + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_u32_unchecked(id: u32) -> Self { + debug_assert!(id < nr_cpu_ids()); + + // Ensure the `id` fits in an [`i32`] as it's also representable that way. + debug_assert!(id <= i32::MAX as u32); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_u32(id: u32) -> Option { + if id >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id)) + } + } + + /// Returns CPU number. + #[inline] + pub fn as_u32(&self) -> u32 { + self.0 + } + + /// Returns the ID of the CPU the code is currently running on. + /// + /// The returned value is considered unstable because it may change + /// unexpectedly due to preemption or CPU migration. It should only be + /// used when the context ensures that the task remains on the same CPU + /// or the users could use a stale (yet valid) CPU ID. + #[inline] + pub fn current() -> Self { + // SAFETY: raw_smp_processor_id() always returns a valid CPU ID. + unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) } + } +} + +impl From for u32 { + fn from(id: CpuId) -> Self { + id.as_u32() + } +} + +impl From for i32 { + fn from(id: CpuId) -> Self { + id.as_u32() as i32 + } +} + +/// Creates a new instance of CPU's device. +/// +/// # Safety +/// +/// Reference counting is not implemented for the CPU device in the C code. When a CPU is +/// hot-unplugged, the corresponding CPU device is unregistered, but its associated memory +/// is not freed. +/// +/// Callers must ensure that the CPU device is not used after it has been unregistered. +/// This can be achieved, for example, by registering a CPU hotplug notifier and removing +/// any references to the CPU device within the notifier's callback. +pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> { + // SAFETY: It is safe to call `get_cpu_device()` for any CPU. + let ptr = unsafe { bindings::get_cpu_device(u32::from(cpu)) }; + if ptr.is_null() { + return Err(ENODEV); + } + + // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to + // a `struct device` and is never freed by the C code. + Ok(unsafe { Device::from_raw(ptr) }) +} diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs new file mode 100644 index 000000000..affa2b949 --- /dev/null +++ b/rust/kernel/cpufreq.rs @@ -0,0 +1,1416 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU frequency scaling. +//! +//! This module provides rust abstractions for interacting with the cpufreq subsystem. +//! +//! C header: [`include/linux/cpufreq.h`](srctree/include/linux/cpufreq.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + cpu::CpuId, + cpumask, + device::{Bound, Device}, + devres, + error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + ffi::{c_char, c_ulong}, + prelude::*, + types::ForeignOwnable, + types::Opaque, +}; + +#[cfg(CONFIG_COMMON_CLK)] +use crate::clk::Clk; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + ops::{Deref, DerefMut}, + pin::Pin, + ptr, +}; + +use macros::vtable; + +/// Maximum length of CPU frequency driver's name. +const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize; + +/// Default transition latency value in nanoseconds. +pub const DEFAULT_TRANSITION_LATENCY_NS: u32 = bindings::CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + +/// CPU frequency driver flags. +pub mod flags { + /// Driver needs to update internal limits even if frequency remains unchanged. + pub const NEED_UPDATE_LIMITS: u16 = 1 << 0; + + /// Platform where constants like `loops_per_jiffy` are unaffected by frequency changes. + pub const CONST_LOOPS: u16 = 1 << 1; + + /// Register driver as a thermal cooling device automatically. + pub const IS_COOLING_DEV: u16 = 1 << 2; + + /// Supports multiple clock domains with per-policy governors in `cpu/cpuN/cpufreq/`. + pub const HAVE_GOVERNOR_PER_POLICY: u16 = 1 << 3; + + /// Allows post-change notifications outside of the `target()` routine. + pub const ASYNC_NOTIFICATION: u16 = 1 << 4; + + /// Ensure CPU starts at a valid frequency from the driver's freq-table. + pub const NEED_INITIAL_FREQ_CHECK: u16 = 1 << 5; + + /// Disallow governors with `dynamic_switching` capability. + pub const NO_AUTO_DYNAMIC_SWITCHING: u16 = 1 << 6; +} + +/// Relations from the C code. +const CPUFREQ_RELATION_L: u32 = 0; +const CPUFREQ_RELATION_H: u32 = 1; +const CPUFREQ_RELATION_C: u32 = 2; + +/// Can be used with any of the above values. +const CPUFREQ_RELATION_E: u32 = 1 << 2; + +/// CPU frequency selection relations. +/// +/// CPU frequency selection relations, each optionally marked as "efficient". +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Relation { + /// Select the lowest frequency at or above target. + Low(bool), + /// Select the highest frequency below or at target. + High(bool), + /// Select the closest frequency to the target. + Close(bool), +} + +impl Relation { + // Construct from a C-compatible `u32` value. + fn new(val: u32) -> Result { + let efficient = val & CPUFREQ_RELATION_E != 0; + + Ok(match val & !CPUFREQ_RELATION_E { + CPUFREQ_RELATION_L => Self::Low(efficient), + CPUFREQ_RELATION_H => Self::High(efficient), + CPUFREQ_RELATION_C => Self::Close(efficient), + _ => return Err(EINVAL), + }) + } +} + +impl From for u32 { + // Convert to a C-compatible `u32` value. + fn from(rel: Relation) -> Self { + let (mut val, efficient) = match rel { + Relation::Low(e) => (CPUFREQ_RELATION_L, e), + Relation::High(e) => (CPUFREQ_RELATION_H, e), + Relation::Close(e) => (CPUFREQ_RELATION_C, e), + }; + + if efficient { + val |= CPUFREQ_RELATION_E; + } + + val + } +} + +/// Policy data. +/// +/// Rust abstraction for the C `struct cpufreq_policy_data`. +/// +/// # Invariants +/// +/// A [`PolicyData`] instance always corresponds to a valid C `struct cpufreq_policy_data`. +/// +/// The callers must ensure that the `struct cpufreq_policy_data` is valid for access and remains +/// valid for the lifetime of the returned reference. +#[repr(transparent)] +pub struct PolicyData(Opaque); + +impl PolicyData { + /// Creates a mutable reference to an existing `struct cpufreq_policy_data` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy_data) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw pointer to the underlying C `cpufreq_policy_data`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_policy_data { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Wrapper for `cpufreq_generic_frequency_table_verify`. + #[inline] + pub fn generic_verify(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_frequency_table_verify(self.as_raw()) }) + } +} + +/// The frequency table index. +/// +/// Represents index with a frequency table. +/// +/// # Invariants +/// +/// The index must correspond to a valid entry in the [`Table`] it is used for. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct TableIndex(usize); + +impl TableIndex { + /// Creates an instance of [`TableIndex`]. + /// + /// # Safety + /// + /// The caller must ensure that `index` correspond to a valid entry in the [`Table`] it is used + /// for. + pub unsafe fn new(index: usize) -> Self { + // INVARIANT: The caller ensures that `index` correspond to a valid entry in the [`Table`]. + Self(index) + } +} + +impl From for usize { + #[inline] + fn from(index: TableIndex) -> Self { + index.0 + } +} + +/// CPU frequency table. +/// +/// Rust abstraction for the C `struct cpufreq_frequency_table`. +/// +/// # Invariants +/// +/// A [`Table`] instance always corresponds to a valid C `struct cpufreq_frequency_table`. +/// +/// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to read a frequency value from [`Table`]. +/// +/// ``` +/// use kernel::cpufreq::{Policy, TableIndex}; +/// +/// fn show_freq(policy: &Policy) -> Result { +/// let table = policy.freq_table()?; +/// +/// // SAFETY: Index is a valid entry in the table. +/// let index = unsafe { TableIndex::new(0) }; +/// +/// pr_info!("The frequency at index 0 is: {:?}\n", table.freq(index)?); +/// pr_info!("The flags at index 0 is: {}\n", table.flags(index)); +/// pr_info!("The data at index 0 is: {}\n", table.data(index)); +/// Ok(()) +/// } +/// ``` +#[repr(transparent)] +pub struct Table(Opaque); + +impl Table { + /// Creates a reference to an existing C `struct cpufreq_frequency_table` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_frequency_table) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Returns the raw mutable pointer to the C `struct cpufreq_frequency_table`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_frequency_table { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Returns frequency at `index` in the [`Table`]. + #[inline] + pub fn freq(&self, index: TableIndex) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + Ok(Hertz::from_khz(unsafe { + (*self.as_raw().add(index.into())).frequency.try_into()? + })) + } + + /// Returns flags at `index` in the [`Table`]. + #[inline] + pub fn flags(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).flags } + } + + /// Returns data at `index` in the [`Table`]. + #[inline] + pub fn data(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).driver_data } + } +} + +/// CPU frequency table owned and pinned in memory, created from a [`TableBuilder`]. +pub struct TableBox { + entries: Pin>, +} + +impl TableBox { + /// Constructs a new [`TableBox`] from a [`KVec`] of entries. + /// + /// # Errors + /// + /// Returns `EINVAL` if the entries list is empty. + #[inline] + fn new(entries: KVec) -> Result { + if entries.is_empty() { + return Err(EINVAL); + } + + Ok(Self { + // Pin the entries to memory, since we are passing its pointer to the C code. + entries: Pin::new(entries), + }) + } + + /// Returns a raw pointer to the underlying C `cpufreq_frequency_table`. + #[inline] + fn as_raw(&self) -> *const bindings::cpufreq_frequency_table { + // The pointer is valid until the table gets dropped. + self.entries.as_ptr() + } +} + +impl Deref for TableBox { + type Target = Table; + + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns TableBox, it is safe to deref. + unsafe { Self::Target::from_raw(self.as_raw()) } + } +} + +/// CPU frequency table builder. +/// +/// This is used by the CPU frequency drivers to build a frequency table dynamically. +/// +/// # Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{TableBuilder, TableIndex}; +/// use kernel::clk::Hertz; +/// +/// let mut builder = TableBuilder::new(); +/// +/// // Adds few entries to the table. +/// builder.add(Hertz::from_mhz(700), 0, 1).unwrap(); +/// builder.add(Hertz::from_mhz(800), 2, 3).unwrap(); +/// builder.add(Hertz::from_mhz(900), 4, 5).unwrap(); +/// builder.add(Hertz::from_ghz(1), 6, 7).unwrap(); +/// +/// let table = builder.to_table().unwrap(); +/// +/// // SAFETY: Index values correspond to valid entries in the table. +/// let (index0, index2) = unsafe { (TableIndex::new(0), TableIndex::new(2)) }; +/// +/// assert_eq!(table.freq(index0), Ok(Hertz::from_mhz(700))); +/// assert_eq!(table.flags(index0), 0); +/// assert_eq!(table.data(index0), 1); +/// +/// assert_eq!(table.freq(index2), Ok(Hertz::from_mhz(900))); +/// assert_eq!(table.flags(index2), 4); +/// assert_eq!(table.data(index2), 5); +/// ``` +#[derive(Default)] +#[repr(transparent)] +pub struct TableBuilder { + entries: KVec, +} + +impl TableBuilder { + /// Creates a new instance of [`TableBuilder`]. + #[inline] + pub fn new() -> Self { + Self { + entries: KVec::new(), + } + } + + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { + // Adds the new entry at the end of the vector. + Ok(self.entries.push( + bindings::cpufreq_frequency_table { + flags, + driver_data, + frequency, + }, + GFP_KERNEL, + )?) + } + + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. + pub fn to_table(mut self) -> Result { + // Add last entry to the table. + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; + + TableBox::new(self.entries) + } +} + +/// CPU frequency policy. +/// +/// Rust abstraction for the C `struct cpufreq_policy`. +/// +/// # Invariants +/// +/// A [`Policy`] instance always corresponds to a valid C `struct cpufreq_policy`. +/// +/// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid +/// for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{DEFAULT_TRANSITION_LATENCY_NS, Policy}; +/// +/// #[allow(clippy::double_parens, reason = "False positive before 1.92.0")] +/// fn update_policy(policy: &mut Policy) { +/// policy +/// .set_dvfs_possible_from_any_cpu(true) +/// .set_fast_switch_possible(true) +/// .set_transition_latency_ns(DEFAULT_TRANSITION_LATENCY_NS); +/// +/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur())); +/// } +/// ``` +#[repr(transparent)] +pub struct Policy(Opaque); + +impl Policy { + /// Creates a reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_policy) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Creates a mutable reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw mutable pointer to the C `struct cpufreq_policy`. + #[inline] + fn as_raw(&self) -> *mut bindings::cpufreq_policy { + let this: *const Self = self; + this.cast_mut().cast() + } + + #[inline] + fn as_ref(&self) -> &bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &*self.as_raw() } + } + + #[inline] + fn as_mut_ref(&mut self) -> &mut bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &mut *self.as_raw() } + } + + /// Returns the primary CPU for the [`Policy`]. + #[inline] + pub fn cpu(&self) -> CpuId { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + unsafe { CpuId::from_u32_unchecked(self.as_ref().cpu) } + } + + /// Returns the minimum frequency for the [`Policy`]. + #[inline] + pub fn min(&self) -> Hertz { + Hertz::from_khz(self.as_ref().min as usize) + } + + /// Set the minimum frequency for the [`Policy`]. + #[inline] + pub fn set_min(&mut self, min: Hertz) -> &mut Self { + self.as_mut_ref().min = min.as_khz() as u32; + self + } + + /// Returns the maximum frequency for the [`Policy`]. + #[inline] + pub fn max(&self) -> Hertz { + Hertz::from_khz(self.as_ref().max as usize) + } + + /// Set the maximum frequency for the [`Policy`]. + #[inline] + pub fn set_max(&mut self, max: Hertz) -> &mut Self { + self.as_mut_ref().max = max.as_khz() as u32; + self + } + + /// Returns the current frequency for the [`Policy`]. + #[inline] + pub fn cur(&self) -> Hertz { + Hertz::from_khz(self.as_ref().cur as usize) + } + + /// Returns the suspend frequency for the [`Policy`]. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + Hertz::from_khz(self.as_ref().suspend_freq as usize) + } + + /// Sets the suspend frequency for the [`Policy`]. + #[inline] + pub fn set_suspend_freq(&mut self, freq: Hertz) -> &mut Self { + self.as_mut_ref().suspend_freq = freq.as_khz() as u32; + self + } + + /// Provides a wrapper to the generic suspend routine. + #[inline] + pub fn generic_suspend(&mut self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_suspend(self.as_mut_ref()) }) + } + + /// Provides a wrapper to the generic get routine. + #[inline] + pub fn generic_get(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + Ok(unsafe { bindings::cpufreq_generic_get(u32::from(self.cpu())) }) + } + + /// Provides a wrapper to the register with energy model using the OPP core. + #[cfg(CONFIG_PM_OPP)] + #[inline] + pub fn register_em_opp(&mut self) { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { bindings::cpufreq_register_em_with_opp(self.as_mut_ref()) }; + } + + /// Gets [`cpumask::Cpumask`] for a cpufreq [`Policy`]. + #[inline] + pub fn cpus(&mut self) -> &mut cpumask::Cpumask { + // SAFETY: The pointer to `cpus` is valid for writing and remains valid for the lifetime of + // the returned reference. + unsafe { cpumask::CpumaskVar::from_raw_mut(&mut self.as_mut_ref().cpus) } + } + + /// Sets clock for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the returned [`Clk`] is not dropped while it is getting used + /// by the C code. + #[cfg(CONFIG_COMMON_CLK)] + pub unsafe fn set_clk(&mut self, dev: &Device, name: Option<&CStr>) -> Result { + let clk = Clk::get(dev, name)?; + self.as_mut_ref().clk = clk.as_raw(); + Ok(clk) + } + + /// Allows / disallows frequency switching code to run on any CPU. + #[inline] + pub fn set_dvfs_possible_from_any_cpu(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().dvfs_possible_from_any_cpu = val; + self + } + + /// Returns if fast switching of frequencies is possible or not. + #[inline] + pub fn fast_switch_possible(&self) -> bool { + self.as_ref().fast_switch_possible + } + + /// Enables / disables fast frequency switching. + #[inline] + pub fn set_fast_switch_possible(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().fast_switch_possible = val; + self + } + + /// Sets transition latency (in nanoseconds) for the [`Policy`]. + #[inline] + pub fn set_transition_latency_ns(&mut self, latency_ns: u32) -> &mut Self { + self.as_mut_ref().cpuinfo.transition_latency = latency_ns; + self + } + + /// Sets cpuinfo `min_freq`. + #[inline] + pub fn set_cpuinfo_min_freq(&mut self, min_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.min_freq = min_freq.as_khz() as u32; + self + } + + /// Sets cpuinfo `max_freq`. + #[inline] + pub fn set_cpuinfo_max_freq(&mut self, max_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.max_freq = max_freq.as_khz() as u32; + self + } + + /// Set `transition_delay_us`, i.e. the minimum time between successive frequency change + /// requests. + #[inline] + pub fn set_transition_delay_us(&mut self, transition_delay_us: u32) -> &mut Self { + self.as_mut_ref().transition_delay_us = transition_delay_us; + self + } + + /// Returns reference to the CPU frequency [`Table`] for the [`Policy`]. + pub fn freq_table(&self) -> Result<&Table> { + if self.as_ref().freq_table.is_null() { + return Err(EINVAL); + } + + // SAFETY: The `freq_table` is guaranteed to be valid for reading and remains valid for the + // lifetime of the returned reference. + Ok(unsafe { Table::from_raw(self.as_ref().freq_table) }) + } + + /// Sets the CPU frequency [`Table`] for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the [`Table`] is not dropped while it is getting used by the + /// C code. + #[inline] + pub unsafe fn set_freq_table(&mut self, table: &Table) -> &mut Self { + self.as_mut_ref().freq_table = table.as_raw(); + self + } + + /// Returns the [`Policy`]'s private data. + pub fn data(&mut self) -> Option<::Borrowed<'_>> { + if self.as_ref().driver_data.is_null() { + None + } else { + // SAFETY: The data is earlier set from [`set_data`]. + Some(unsafe { T::borrow(self.as_ref().driver_data.cast()) }) + } + } + + /// Sets the private data of the [`Policy`] using a foreign-ownable wrapper. + /// + /// # Errors + /// + /// Returns `EBUSY` if private data is already set. + fn set_data(&mut self, data: T) -> Result { + if self.as_ref().driver_data.is_null() { + // Transfer the ownership of the data to the foreign interface. + self.as_mut_ref().driver_data = ::into_foreign(data).cast(); + Ok(()) + } else { + Err(EBUSY) + } + } + + /// Clears and returns ownership of the private data. + fn clear_data(&mut self) -> Option { + if self.as_ref().driver_data.is_null() { + None + } else { + let data = Some( + // SAFETY: The data is earlier set by us from [`set_data`]. It is safe to take + // back the ownership of the data from the foreign interface. + unsafe { ::from_foreign(self.as_ref().driver_data.cast()) }, + ); + self.as_mut_ref().driver_data = ptr::null_mut(); + data + } + } +} + +/// CPU frequency policy created from a CPU number. +/// +/// This struct represents the CPU frequency policy obtained for a specific CPU, providing safe +/// access to the underlying `cpufreq_policy` and ensuring proper cleanup when the `PolicyCpu` is +/// dropped. +struct PolicyCpu<'a>(&'a mut Policy); + +impl<'a> PolicyCpu<'a> { + fn from_cpu(cpu: CpuId) -> Result { + // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. + let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?; + + Ok(Self( + // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of + // the returned reference. + unsafe { Policy::from_raw_mut(ptr) }, + )) + } +} + +impl<'a> Deref for PolicyCpu<'a> { + type Target = Policy; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a> DerefMut for PolicyCpu<'a> { + fn deref_mut(&mut self) -> &mut Policy { + self.0 + } +} + +impl<'a> Drop for PolicyCpu<'a> { + fn drop(&mut self) { + // SAFETY: The underlying pointer is guaranteed to be valid for the lifetime of `self`. + unsafe { bindings::cpufreq_cpu_put(self.0.as_raw()) }; + } +} + +/// CPU frequency driver. +/// +/// Implement this trait to provide a CPU frequency driver and its callbacks. +/// +/// Reference: +#[vtable] +pub trait Driver { + /// Driver's name. + const NAME: &'static CStr; + + /// Driver's flags. + const FLAGS: u16; + + /// Boost support. + const BOOST_ENABLED: bool; + + /// Policy specific data. + /// + /// Require that `PData` implements `ForeignOwnable`. We guarantee to never move the underlying + /// wrapped data structure. + type PData: ForeignOwnable; + + /// Driver's `init` callback. + fn init(policy: &mut Policy) -> Result; + + /// Driver's `exit` callback. + fn exit(_policy: &mut Policy, _data: Option) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `online` callback. + fn online(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `offline` callback. + fn offline(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `suspend` callback. + fn suspend(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `resume` callback. + fn resume(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `ready` callback. + fn ready(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `verify` callback. + fn verify(data: &mut PolicyData) -> Result; + + /// Driver's `setpolicy` callback. + fn setpolicy(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target` callback. + fn target(_policy: &mut Policy, _target_freq: u32, _relation: Relation) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_index` callback. + fn target_index(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `fast_switch` callback. + fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `adjust_perf` callback. + fn adjust_perf( + _policy: &mut Policy, + _min_perf: usize, + _target_perf: usize, + _max_perf: usize, + _capacity: usize, + ) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get_intermediate` callback. + fn get_intermediate(_policy: &mut Policy, _index: TableIndex) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_intermediate` callback. + fn target_intermediate(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get` callback. + fn get(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `update_limits` callback. + fn update_limits(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `bios_limit` callback. + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `set_boost` callback. + fn set_boost(_policy: &mut Policy, _state: i32) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `register_em` callback. + fn register_em(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// CPU frequency driver Registration. +/// +/// # Examples +/// +/// The following example demonstrates how to register a cpufreq driver. +/// +/// ``` +/// use kernel::{ +/// cpufreq, +/// device::{Core, Device}, +/// macros::vtable, +/// of, platform, +/// sync::Arc, +/// }; +/// struct SampleDevice; +/// +/// #[derive(Default)] +/// struct SampleDriver; +/// +/// #[vtable] +/// impl cpufreq::Driver for SampleDriver { +/// const NAME: &'static CStr = c"cpufreq-sample"; +/// const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV; +/// const BOOST_ENABLED: bool = true; +/// +/// type PData = Arc; +/// +/// fn init(policy: &mut cpufreq::Policy) -> Result { +/// // Initialize here +/// Ok(Arc::new(SampleDevice, GFP_KERNEL)?) +/// } +/// +/// fn exit(_policy: &mut cpufreq::Policy, _data: Option) -> Result { +/// Ok(()) +/// } +/// +/// fn suspend(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_suspend() +/// } +/// +/// fn verify(data: &mut cpufreq::PolicyData) -> Result { +/// data.generic_verify() +/// } +/// +/// fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result { +/// // Update CPU frequency +/// Ok(()) +/// } +/// +/// fn get(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_get() +/// } +/// } +/// +/// impl platform::Driver for SampleDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = None; +/// +/// fn probe<'bound>( +/// pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit + 'bound { +/// cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; +/// Ok(Self {}) +/// } +/// } +/// ``` +#[repr(transparent)] +pub struct Registration(KBox>, PhantomData); + +/// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads +/// or CPUs, so it is safe to share it. +unsafe impl Sync for Registration {} + +#[allow(clippy::non_send_fields_in_send_ty)] +/// SAFETY: Registration with and unregistration from the cpufreq subsystem can happen from any +/// thread. +unsafe impl Send for Registration {} + +impl Registration { + const VTABLE: bindings::cpufreq_driver = bindings::cpufreq_driver { + name: Self::copy_name(T::NAME), + boost_enabled: T::BOOST_ENABLED, + flags: T::FLAGS, + + // Initialize mandatory callbacks. + init: Some(Self::init_callback), + verify: Some(Self::verify_callback), + + // Initialize optional callbacks based on the traits of `T`. + setpolicy: if T::HAS_SETPOLICY { + Some(Self::setpolicy_callback) + } else { + None + }, + target: if T::HAS_TARGET { + Some(Self::target_callback) + } else { + None + }, + target_index: if T::HAS_TARGET_INDEX { + Some(Self::target_index_callback) + } else { + None + }, + fast_switch: if T::HAS_FAST_SWITCH { + Some(Self::fast_switch_callback) + } else { + None + }, + adjust_perf: if T::HAS_ADJUST_PERF { + Some(Self::adjust_perf_callback) + } else { + None + }, + get_intermediate: if T::HAS_GET_INTERMEDIATE { + Some(Self::get_intermediate_callback) + } else { + None + }, + target_intermediate: if T::HAS_TARGET_INTERMEDIATE { + Some(Self::target_intermediate_callback) + } else { + None + }, + get: if T::HAS_GET { + Some(Self::get_callback) + } else { + None + }, + update_limits: if T::HAS_UPDATE_LIMITS { + Some(Self::update_limits_callback) + } else { + None + }, + bios_limit: if T::HAS_BIOS_LIMIT { + Some(Self::bios_limit_callback) + } else { + None + }, + online: if T::HAS_ONLINE { + Some(Self::online_callback) + } else { + None + }, + offline: if T::HAS_OFFLINE { + Some(Self::offline_callback) + } else { + None + }, + exit: if T::HAS_EXIT { + Some(Self::exit_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Self::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Self::resume_callback) + } else { + None + }, + ready: if T::HAS_READY { + Some(Self::ready_callback) + } else { + None + }, + set_boost: if T::HAS_SET_BOOST { + Some(Self::set_boost_callback) + } else { + None + }, + register_em: if T::HAS_REGISTER_EM { + Some(Self::register_em_callback) + } else { + None + }, + ..pin_init::zeroed() + }; + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { + let src = name.to_bytes_with_nul(); + let mut dst = [0; CPUFREQ_NAME_LEN]; + + build_assert!(src.len() <= CPUFREQ_NAME_LEN); + + let mut i = 0; + while i < src.len() { + dst[i] = src[i]; + i += 1; + } + + dst + } + + /// Registers a CPU frequency driver with the cpufreq core. + pub fn new() -> Result { + // We can't use `&Self::VTABLE` directly because the cpufreq core modifies some fields in + // the C `struct cpufreq_driver`, which requires a mutable reference. + let mut drv = KBox::new(UnsafeCell::new(Self::VTABLE), GFP_KERNEL)?; + + // SAFETY: `drv` is guaranteed to be valid for the lifetime of `Registration`. + to_result(unsafe { bindings::cpufreq_register_driver(drv.get_mut()) })?; + + Ok(Self(drv, PhantomData)) + } + + /// Same as [`Registration::new`], but does not return a [`Registration`] instance. + /// + /// Instead the [`Registration`] is owned by [`devres::register`] and will be dropped, once the + /// device is detached. + pub fn new_foreign_owned(dev: &Device) -> Result + where + T: 'static, + { + devres::register(dev, Self::new()?, GFP_KERNEL) + } +} + +/// CPU frequency driver callbacks. +impl Registration { + /// Driver's `init` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = T::init(policy)?; + policy.set_data(data)?; + Ok(0) + }) + } + + /// Driver's `exit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = policy.clear_data(); + let _ = T::exit(policy, data); + } + + /// Driver's `online` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::online(policy).map(|()| 0) + }) + } + + /// Driver's `offline` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::offline(policy).map(|()| 0) + }) + } + + /// Driver's `suspend` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::suspend(policy).map(|()| 0) + }) + } + + /// Driver's `resume` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::resume(policy).map(|()| 0) + }) + } + + /// Driver's `ready` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::ready(policy); + } + + /// Driver's `verify` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let data = unsafe { PolicyData::from_raw_mut(ptr) }; + T::verify(data).map(|()| 0) + }) + } + + /// Driver's `setpolicy` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::setpolicy(policy).map(|()| 0) + }) + } + + /// Driver's `target` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: c_uint, + relation: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::target(policy, target_freq, Relation::new(relation)?).map(|()| 0) + }) + } + + /// Driver's `target_index` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_index_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_index(policy, index).map(|()| 0) + }) + } + + /// Driver's `fast_switch` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn fast_switch_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: c_uint, + ) -> c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::fast_switch(policy, target_freq) + } + + /// Driver's `adjust_perf` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn adjust_perf_callback( + ptr: *mut bindings::cpufreq_policy, + min_perf: c_ulong, + target_perf: c_ulong, + max_perf: c_ulong, + capacity: c_ulong, + ) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity); + } + + /// Driver's `get_intermediate` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn get_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::get_intermediate(policy, index) + } + + /// Driver's `target_intermediate` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_intermediate(policy, index).map(|()| 0) + }) + } + + /// Driver's `get` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + unsafe extern "C" fn get_callback(cpu: c_uint) -> c_uint { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; + + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).unwrap_or(0)) + } + + /// Driver's `update_limit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::update_limits(policy); + } + + /// Driver's `bios_limit` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn bios_limit_callback(cpu: c_int, limit: *mut c_uint) -> c_int { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_i32_unchecked(cpu) }; + + from_result(|| { + let mut policy = PolicyCpu::from_cpu(cpu_id)?; + let val = T::bios_limit(&mut policy)?; + // SAFETY: `limit` is guaranteed by the C code to be valid. + unsafe { + *limit = val; + } + Ok(0) + }) + } + + /// Driver's `set_boost` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn set_boost_callback( + ptr: *mut bindings::cpufreq_policy, + state: c_int, + ) -> c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::set_boost(policy, state).map(|()| 0) + }) + } + + /// Driver's `register_em` callback. + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::register_em(policy); + } +} + +impl Drop for Registration { + /// Unregisters with the cpufreq core. + fn drop(&mut self) { + // SAFETY: `self.0` is guaranteed to be valid for the lifetime of `Registration`. + unsafe { bindings::cpufreq_unregister_driver(self.0.get_mut()) }; + } +} diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs new file mode 100644 index 000000000..44bb36636 --- /dev/null +++ b/rust/kernel/cpumask.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU Mask abstractions. +//! +//! C header: [`include/linux/cpumask.h`](srctree/include/linux/cpumask.h) + +use crate::{ + alloc::{AllocError, Flags}, + cpu::CpuId, + prelude::*, + types::Opaque, +}; + +#[cfg(CONFIG_CPUMASK_OFFSTACK)] +use core::ptr::{self, NonNull}; + +use core::ops::{Deref, DerefMut}; + +/// A CPU Mask. +/// +/// Rust abstraction for the C `struct cpumask`. +/// +/// # Invariants +/// +/// A [`Cpumask`] instance always corresponds to a valid C `struct cpumask`. +/// +/// The callers must ensure that the `struct cpumask` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// # Examples +/// +/// The following example demonstrates how to update a [`Cpumask`]. +/// +/// ``` +/// use kernel::bindings; +/// use kernel::cpu::CpuId; +/// use kernel::cpumask::Cpumask; +/// +/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: CpuId, clear_cpu: CpuId) { +/// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the +/// // returned reference. +/// let mask = unsafe { Cpumask::from_raw_mut(ptr) }; +/// +/// mask.set(set_cpu); +/// mask.clear(clear_cpu); +/// } +/// ``` +#[repr(transparent)] +pub struct Cpumask(Opaque); + +impl Cpumask { + /// Creates a mutable reference from an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference from an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Obtain the raw `struct cpumask` pointer. + pub fn as_raw(&self) -> *mut bindings::cpumask { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Set `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `set()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_set_cpu()`. + #[inline] + pub fn set(&mut self, cpu: CpuId) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`. + unsafe { bindings::__cpumask_set_cpu(u32::from(cpu), self.as_raw()) }; + } + + /// Clear `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `clear()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_clear_cpu()`. + #[inline] + pub fn clear(&mut self, cpu: CpuId) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to + // `__cpumask_clear_cpu`. + unsafe { bindings::__cpumask_clear_cpu(i32::from(cpu), self.as_raw()) }; + } + + /// Test `cpu` in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_test_cpu` API. + #[inline] + pub fn test(&self, cpu: CpuId) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`. + unsafe { bindings::cpumask_test_cpu(i32::from(cpu), self.as_raw()) } + } + + /// Set all CPUs in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_setall` API. + #[inline] + pub fn setall(&mut self) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_setall`. + unsafe { bindings::cpumask_setall(self.as_raw()) }; + } + + /// Checks if cpumask is empty. + /// + /// Equivalent to the kernel's `cpumask_empty` API. + #[inline] + pub fn empty(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_empty`. + unsafe { bindings::cpumask_empty(self.as_raw()) } + } + + /// Checks if cpumask is full. + /// + /// Equivalent to the kernel's `cpumask_full` API. + #[inline] + pub fn full(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_full`. + unsafe { bindings::cpumask_full(self.as_raw()) } + } + + /// Get weight of the cpumask. + /// + /// Equivalent to the kernel's `cpumask_weight` API. + #[inline] + pub fn weight(&self) -> u32 { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_weight`. + unsafe { bindings::cpumask_weight(self.as_raw()) } + } + + /// Copy cpumask. + /// + /// Equivalent to the kernel's `cpumask_copy` API. + #[inline] + pub fn copy(&self, dstp: &mut Self) { + // SAFETY: By the type invariant, `Self::as_raw` is a valid argument to `cpumask_copy`. + unsafe { bindings::cpumask_copy(dstp.as_raw(), self.as_raw()) }; + } +} + +/// A CPU Mask pointer. +/// +/// Rust abstraction for the C `struct cpumask_var_t`. +/// +/// # Invariants +/// +/// A [`CpumaskVar`] instance always corresponds to a valid C `struct cpumask_var_t`. +/// +/// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid +/// for the lifetime of [`CpumaskVar`]. +/// +/// # Examples +/// +/// The following example demonstrates how to create and update a [`CpumaskVar`]. +/// +/// ``` +/// use kernel::cpu::CpuId; +/// use kernel::cpumask::CpumaskVar; +/// +/// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap(); +/// +/// assert!(mask.empty()); +/// let mut count = 0; +/// +/// let cpu2 = CpuId::from_u32(2); +/// if let Some(cpu) = cpu2 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// let cpu3 = CpuId::from_u32(3); +/// if let Some(cpu) = cpu3 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// assert_eq!(mask.weight(), count); +/// +/// let mask2 = CpumaskVar::try_clone(&mask).unwrap(); +/// +/// if let Some(cpu) = cpu2 { +/// assert!(mask2.test(cpu)); +/// } +/// +/// if let Some(cpu) = cpu3 { +/// assert!(mask2.test(cpu)); +/// } +/// assert_eq!(mask2.weight(), count); +/// ``` +#[repr(transparent)] +pub struct CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: NonNull, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask, +} + +impl CpumaskVar { + /// Creates a zero-initialized instance of the [`CpumaskVar`]. + pub fn new_zero(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::zalloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask(Opaque::zeroed()), + }) + } + + /// Creates an instance of the [`CpumaskVar`]. + /// + /// # Safety + /// + /// The caller must ensure that the returned [`CpumaskVar`] is properly initialized before + /// getting used. + pub unsafe fn new(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::alloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask(Opaque::uninit()), + }) + } + + /// Creates a mutable reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask_var_t) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask_var_t) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Clones cpumask. + pub fn try_clone(cpumask: &Cpumask) -> Result { + // SAFETY: The returned cpumask_var is initialized right after this call. + let mut cpumask_var = unsafe { Self::new(GFP_KERNEL) }?; + + cpumask.copy(&mut cpumask_var); + Ok(cpumask_var) + } +} + +// Make [`CpumaskVar`] behave like a pointer to [`Cpumask`]. +impl Deref for CpumaskVar { + type Target = Cpumask; + + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { &*self.ptr.as_ptr() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref(&self) -> &Self::Target { + &self.mask + } +} + +impl DerefMut for CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref_mut(&mut self) -> &mut Cpumask { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { self.ptr.as_mut() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref_mut(&mut self) -> &mut Cpumask { + &mut self.mask + } +} + +impl Drop for CpumaskVar { + fn drop(&mut self) { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `free_cpumask_var`. + unsafe { + bindings::free_cpumask_var(self.as_raw()) + }; + } +} diff --git a/rust/kernel/cred.rs b/rust/kernel/cred.rs new file mode 100644 index 000000000..ffa156b9d --- /dev/null +++ b/rust/kernel/cred.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Credentials management. +//! +//! C header: [`include/linux/cred.h`](srctree/include/linux/cred.h). +//! +//! Reference: + +use crate::{bindings, sync::aref::AlwaysRefCounted, task::Kuid, types::Opaque}; + +/// Wraps the kernel's `struct cred`. +/// +/// Credentials are used for various security checks in the kernel. +/// +/// Most fields of credentials are immutable. When things have their credentials changed, that +/// happens by replacing the credential instead of changing an existing credential. See the [kernel +/// documentation][ref] for more info on this. +/// +/// # Invariants +/// +/// Instances of this type are always ref-counted, that is, a call to `get_cred` ensures that the +/// allocation remains valid at least until the matching call to `put_cred`. +/// +/// [ref]: https://www.kernel.org/doc/html/latest/security/credentials.html +#[repr(transparent)] +pub struct Credential(Opaque); + +// SAFETY: +// - `Credential::dec_ref` can be called from any thread. +// - It is okay to send ownership of `Credential` across thread boundaries. +unsafe impl Send for Credential {} + +// SAFETY: It's OK to access `Credential` through shared references from other threads because +// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for Credential {} + +impl Credential { + /// Creates a reference to a [`Credential`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Credential`] reference. + #[inline] + pub unsafe fn from_ptr<'a>(ptr: *const bindings::cred) -> &'a Credential { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Credential` type being transparent makes the cast ok. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to the inner credential. + #[inline] + pub fn as_ptr(&self) -> *const bindings::cred { + self.0.get() + } + + /// Get the id for this security context. + #[inline] + pub fn get_secid(&self) -> u32 { + let mut secid = 0; + // SAFETY: The invariants of this type ensures that the pointer is valid. + unsafe { bindings::security_cred_getsecid(self.0.get(), &mut secid) }; + secid + } + + /// Returns the effective UID of the given credential. + #[inline] + pub fn euid(&self) -> Kuid { + // SAFETY: By the type invariant, we know that `self.0` is valid. Furthermore, the `euid` + // field of a credential is never changed after initialization, so there is no potential + // for data races. + Kuid::from_raw(unsafe { (*self.0.get()).euid }) + } +} + +// SAFETY: The type invariants guarantee that `Credential` is always ref-counted. +unsafe impl AlwaysRefCounted for Credential { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_cred(self.0.get()) }; + } + + #[inline] + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. The cast is okay + // because `Credential` has the same representation as `struct cred`. + unsafe { bindings::put_cred(obj.cast().as_ptr()) }; + } +} diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs new file mode 100644 index 000000000..d7b8014a6 --- /dev/null +++ b/rust/kernel/debugfs.rs @@ -0,0 +1,724 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! DebugFS Abstraction +//! +//! C header: [`include/linux/debugfs.h`](srctree/include/linux/debugfs.h) + +// When DebugFS is disabled, many parameters are dead. Linting for this isn't helpful. +#![cfg_attr(not(CONFIG_DEBUG_FS), allow(unused_variables))] + +#[cfg(CONFIG_DEBUG_FS)] +use crate::sync::Arc; +use crate::{ + fmt, + prelude::*, + str::CStr, + uaccess::UserSliceReader, // +}; + +#[cfg(CONFIG_DEBUG_FS)] +use core::mem::ManuallyDrop; +use core::{ + marker::{ + PhantomData, + PhantomPinned, // + }, + ops::Deref, +}; + +mod traits; +pub use traits::{ + BinaryReader, + BinaryReaderMut, + BinaryWriter, + Reader, + Writer, // +}; + +mod callback_adapters; +use callback_adapters::{ + FormatAdapter, + NoWriter, + WritableAdapter, // +}; + +mod file_ops; +use file_ops::{ + BinaryReadFile, + BinaryReadWriteFile, + BinaryWriteFile, + FileOps, + ReadFile, + ReadWriteFile, + WriteFile, // +}; + +#[cfg(CONFIG_DEBUG_FS)] +mod entry; +#[cfg(CONFIG_DEBUG_FS)] +use entry::Entry; + +/// Owning handle to a DebugFS directory. +/// +/// The directory in the filesystem represented by [`Dir`] will be removed when handle has been +/// dropped *and* all children have been removed. +// If we have a parent, we hold a reference to it in the `Entry`. This prevents the `dentry` +// we point to from being cleaned up if our parent `Dir`/`Entry` is dropped before us. +// +// The `None` option indicates that the `Arc` could not be allocated, so our children would not be +// able to refer to us. In this case, we need to silently fail. All future child directories/files +// will silently fail as well. +#[derive(Clone)] +pub struct Dir(#[cfg(CONFIG_DEBUG_FS)] Option>>); + +impl Dir { + /// Create a new directory in DebugFS. If `parent` is [`None`], it will be created at the root. + fn create(name: &CStr, parent: Option<&Dir>) -> Self { + #[cfg(CONFIG_DEBUG_FS)] + { + let parent_entry = match parent { + // If the parent couldn't be allocated, just early-return + Some(Dir(None)) => return Self(None), + Some(Dir(Some(entry))) => Some(entry.clone()), + None => None, + }; + Self( + // If Arc creation fails, the `Entry` will be dropped, so the directory will be + // cleaned up. + Arc::new(Entry::dynamic_dir(name, parent_entry), GFP_KERNEL).ok(), + ) + } + #[cfg(not(CONFIG_DEBUG_FS))] + Self() + } + + /// Creates a DebugFS file which will own the data produced by the initializer provided in + /// `data`. + fn create_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + file_ops: &'static FileOps, + ) -> impl PinInit, E> + 'a + where + T: Sync + 'static, + { + let scope = Scope::::new(data, move |data| { + #[cfg(CONFIG_DEBUG_FS)] + if let Some(parent) = &self.0 { + // SAFETY: Because data derives from a scope, and our entry will be dropped before + // the data is dropped, it is guaranteed to outlive the entry we return. + unsafe { Entry::dynamic_file(name, parent.clone(), data, file_ops) } + } else { + Entry::empty() + } + }); + try_pin_init! { + File { + scope <- scope + } ? E + } + } + + /// Create a new directory in DebugFS at the root. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// let debugfs = Dir::new(c"parent"); + /// ``` + pub fn new(name: &CStr) -> Self { + Dir::create(name, None) + } + + /// Creates a subdirectory within this directory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// let parent = Dir::new(c"parent"); + /// let child = parent.subdir(c"child"); + /// ``` + pub fn subdir(&self, name: &CStr) -> Self { + Dir::create(name, Some(self)) + } + + /// Creates a read-only file in this directory. + /// + /// The file's contents are produced by invoking [`Writer::write`] on the value initialized by + /// `data`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// # use kernel::prelude::*; + /// # let dir = Dir::new(c"my_debugfs_dir"); + /// let file = KBox::pin_init(dir.read_only_file(c"foo", 200), GFP_KERNEL)?; + /// // "my_debugfs_dir/foo" now contains the number 200. + /// // The file is removed when `file` is dropped. + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_only_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Writer + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`] on the value + /// initialized by `data`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::debugfs::Dir; + /// # use kernel::prelude::*; + /// # let dir = Dir::new(c"my_debugfs_dir"); + /// let file = KBox::pin_init(dir.read_binary_file(c"foo", [0x1, 0x2]), GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryWriter + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only file in this directory, with contents from a callback. + /// + /// `f` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{ + /// # debugfs::Dir, + /// # prelude::*, + /// # sync::atomic::{ + /// # Atomic, + /// # Relaxed, + /// # }, + /// # }; + /// # let dir = Dir::new(c"foo"); + /// let file = KBox::pin_init( + /// dir.read_callback_file(c"bar", + /// Atomic::::new(3), + /// &|val, f| { + /// let out = val.load(Relaxed); + /// writeln!(f, "{out:#010x}") + /// }), + /// GFP_KERNEL)?; + /// // Reading "foo/bar" will show "0x00000003". + /// file.store(10, Relaxed); + /// // Reading "foo/bar" will now show "0x0000000a". + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_callback_file<'a, T, E: 'a, F>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _f: &'static F, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + { + let file_ops = >::FILE_OPS.adapt(); + self.create_file(name, data, file_ops) + } + + /// Creates a read-write file in this directory. + /// + /// Reading the file uses the [`Writer`] implementation. + /// Writing to the file uses the [`Reader`] implementation. + pub fn read_write_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Writer + Reader + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation. + /// Writing to the file uses the [`BinaryReader`] implementation. + pub fn read_write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryWriter + BinaryReader + Send + Sync + 'static, + { + let file_ops = &>::FILE_OPS; + self.create_file(name, data, file_ops) + } + + /// Creates a read-write file in this directory, with logic from callbacks. + /// + /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. + /// + /// `f` and `w` must be function items or non-capturing closures. + /// This is statically asserted and not a safety requirement. + pub fn read_write_callback_file<'a, T, E: 'a, F, W>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _f: &'static F, + _w: &'static W, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let file_ops = + , W> as file_ops::ReadWriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, file_ops) + } + + /// Creates a write-only file in this directory. + /// + /// The file owns its backing data. Writing to the file uses the [`Reader`] + /// implementation. + /// + /// The file is removed when the returned [`File`] is dropped. + pub fn write_only_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: Reader + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only binary file in this directory. + /// + /// The file owns its backing data. Writing to the file uses the [`BinaryReader`] + /// implementation. + /// + /// The file is removed when the returned [`File`] is dropped. + pub fn write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + ) -> impl PinInit, E> + 'a + where + T: BinaryReader + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only file in this directory, with write logic from a callback. + /// + /// `w` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + pub fn write_callback_file<'a, T, E: 'a, W>( + &'a self, + name: &'a CStr, + data: impl PinInit + 'a, + _w: &'static W, + ) -> impl PinInit, E> + 'a + where + T: Send + Sync + 'static, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let file_ops = , W> as WriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, file_ops) + } + + // While this function is safe, it is intentionally not public because it's a bit of a + // footgun. + // + // Unless you also extract the `entry` later and schedule it for `Drop` at the appropriate + // time, a `ScopedDir` with a `Dir` parent will never be deleted. + fn scoped_dir<'data>(&self, name: &CStr) -> ScopedDir<'data, 'static> { + #[cfg(CONFIG_DEBUG_FS)] + { + let parent_entry = match &self.0 { + None => return ScopedDir::empty(), + Some(entry) => entry.clone(), + }; + ScopedDir { + entry: ManuallyDrop::new(Entry::dynamic_dir(name, Some(parent_entry))), + _phantom: PhantomData, + } + } + #[cfg(not(CONFIG_DEBUG_FS))] + ScopedDir::empty() + } + + /// Creates a new scope, which is a directory associated with some data `T`. + /// + /// The created directory will be a subdirectory of `self`. The `init` closure is called to + /// populate the directory with files and subdirectories. These files can reference the data + /// stored in the scope. + /// + /// The entire directory tree created within the scope will be removed when the returned + /// `Scope` handle is dropped. + pub fn scope<'a, T: 'a, E: 'a, F>( + &'a self, + data: impl PinInit + 'a, + name: &'a CStr, + init: F, + ) -> impl PinInit, E> + 'a + where + F: for<'data, 'dir> FnOnce(&'data T, &'dir ScopedDir<'data, 'dir>) + 'a, + { + Scope::new(data, |data| { + let scoped = self.scoped_dir(name); + init(data, &scoped); + scoped.into_entry() + }) + } +} + +#[pin_data] +/// Handle to a DebugFS scope, which ensures that attached `data` will outlive the DebugFS entry +/// without moving. +/// +/// This is internally used to back [`File`], and used in the API to represent the attachment +/// of a directory lifetime to a data structure which may be jointly accessed by a number of +/// different files. +/// +/// When dropped, a `Scope` will remove all directories and files in the filesystem backed by the +/// attached data structure prior to releasing the attached data. +pub struct Scope { + // This order is load-bearing for drops - `_entry` must be dropped before `data`. + #[cfg(CONFIG_DEBUG_FS)] + _entry: Entry<'static>, + #[pin] + data: T, + // Even if `T` is `Unpin`, we still can't allow it to be moved. + #[pin] + _pin: PhantomPinned, +} + +#[pin_data] +/// Handle to a DebugFS file, owning its backing data. +/// +/// When dropped, the DebugFS file will be removed and the attached data will be dropped. +pub struct File { + #[pin] + scope: Scope, +} + +#[cfg(not(CONFIG_DEBUG_FS))] +impl<'b, T: 'b> Scope { + fn new(data: impl PinInit + 'b, init: F) -> impl PinInit + 'b + where + F: for<'a> FnOnce(&'a T) + 'b, + { + try_pin_init! { + Self { + data <- data, + _pin: PhantomPinned + } ? E + } + .pin_chain(|scope| { + init(&scope.data); + Ok(()) + }) + } +} + +#[cfg(CONFIG_DEBUG_FS)] +impl<'b, T: 'b> Scope { + fn entry_mut(self: Pin<&mut Self>) -> &mut Entry<'static> { + // SAFETY: _entry is not structurally pinned. + unsafe { &mut Pin::into_inner_unchecked(self)._entry } + } + + fn new(data: impl PinInit + 'b, init: F) -> impl PinInit + 'b + where + F: for<'a> FnOnce(&'a T) -> Entry<'static> + 'b, + { + try_pin_init! { + Self { + _entry: Entry::empty(), + data <- data, + _pin: PhantomPinned + } ? E + } + .pin_chain(|scope| { + *scope.entry_mut() = init(&scope.data); + Ok(()) + }) + } +} + +impl<'a, T: 'a> Scope { + /// Creates a new scope, which is a directory at the root of the debugfs filesystem, + /// associated with some data `T`. + /// + /// The `init` closure is called to populate the directory with files and subdirectories. These + /// files can reference the data stored in the scope. + /// + /// The entire directory tree created within the scope will be removed when the returned + /// `Scope` handle is dropped. + pub fn dir( + data: impl PinInit + 'a, + name: &'a CStr, + init: F, + ) -> impl PinInit + 'a + where + F: for<'data, 'dir> FnOnce(&'data T, &'dir ScopedDir<'data, 'dir>) + 'a, + { + Scope::new(data, |data| { + let scoped = ScopedDir::new(name); + init(data, &scoped); + scoped.into_entry() + }) + } +} + +impl Deref for Scope { + type Target = T; + fn deref(&self) -> &T { + &self.data + } +} + +impl Deref for File { + type Target = T; + fn deref(&self) -> &T { + &self.scope + } +} + +/// A handle to a directory which will live at most `'dir`, accessing data that will live for at +/// least `'data`. +/// +/// Dropping a ScopedDir will not delete or clean it up, this is expected to occur through dropping +/// the `Scope` that created it. +pub struct ScopedDir<'data, 'dir> { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop>, + _phantom: PhantomData &'dir ()>, +} + +impl<'data, 'dir> ScopedDir<'data, 'dir> { + /// Creates a subdirectory inside this `ScopedDir`. + /// + /// The returned directory handle cannot outlive this one. + pub fn dir<'dir2>(&'dir2 self, name: &CStr) -> ScopedDir<'data, 'dir2> { + #[cfg(not(CONFIG_DEBUG_FS))] + let _ = name; + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::dir(name, Some(&*self.entry))), + _phantom: PhantomData, + } + } + + fn create_file(&self, name: &CStr, data: &'data T, vtable: &'static FileOps) { + #[cfg(CONFIG_DEBUG_FS)] + core::mem::forget(Entry::file(name, &self.entry, data, vtable)); + } + + /// Creates a read-only file in this directory. + /// + /// The file's contents are produced by invoking [`Writer::write`]. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_only_file(&self, name: &CStr, data: &'data T) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`]. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a read-only file in this directory, with contents from a callback. + /// + /// The file contents are generated by calling `f` with `data`. + /// + /// + /// `f` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_callback_file(&self, name: &CStr, data: &'data T, _f: &'static F) + where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + { + let vtable = as ReadFile<_>>::FILE_OPS.adapt(); + self.create_file(name, data, vtable) + } + + /// Creates a read-write file in this directory. + /// + /// Reading the file uses the [`Writer`] implementation on `data`. Writing to the file uses + /// the [`Reader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_write_file( + &self, + name: &CStr, + data: &'data T, + ) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation on `data`. Writing to the file + /// uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_write_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a read-write file in this directory, with logic from callbacks. + /// + /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. + /// + /// `f` and `w` must be function items or non-capturing closures. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn read_write_callback_file( + &self, + name: &CStr, + data: &'data T, + _f: &'static F, + _w: &'static W, + ) where + T: Send + Sync + 'static, + F: Fn(&T, &mut fmt::Formatter<'_>) -> fmt::Result + Send + Sync, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let vtable = , W> as ReadWriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, vtable) + } + + /// Creates a write-only file in this directory. + /// + /// Writing to the file uses the [`Reader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn write_only_file(&self, name: &CStr, data: &'data T) { + let vtable = &>::FILE_OPS; + self.create_file(name, data, vtable) + } + + /// Creates a write-only binary file in this directory. + /// + /// Writing to the file uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn write_binary_file( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + + /// Creates a write-only file in this directory, with write logic from a callback. + /// + /// Writing to the file is handled by `w`. + /// + /// `w` must be a function item or a non-capturing closure. + /// This is statically asserted and not a safety requirement. + /// + /// This function does not produce an owning handle to the file. The created + /// file is removed when the [`Scope`] that this directory belongs + /// to is dropped. + pub fn write_only_callback_file(&self, name: &CStr, data: &'data T, _w: &'static W) + where + T: Send + Sync + 'static, + W: Fn(&T, &mut UserSliceReader) -> Result + Send + Sync, + { + let vtable = &, W> as WriteFile<_>>::FILE_OPS + .adapt() + .adapt(); + self.create_file(name, data, vtable) + } + + fn empty() -> Self { + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::empty()), + _phantom: PhantomData, + } + } + #[cfg(CONFIG_DEBUG_FS)] + fn into_entry(self) -> Entry<'dir> { + ManuallyDrop::into_inner(self.entry) + } + #[cfg(not(CONFIG_DEBUG_FS))] + fn into_entry(self) {} +} + +impl<'data> ScopedDir<'data, 'static> { + // This is safe, but intentionally not exported due to footgun status. A ScopedDir with no + // parent will never be released by default, and needs to have its entry extracted and used + // somewhere. + fn new(name: &CStr) -> ScopedDir<'data, 'static> { + ScopedDir { + #[cfg(CONFIG_DEBUG_FS)] + entry: ManuallyDrop::new(Entry::dir(name, None)), + _phantom: PhantomData, + } + } +} diff --git a/rust/kernel/debugfs/callback_adapters.rs b/rust/kernel/debugfs/callback_adapters.rs new file mode 100644 index 000000000..dee7d021e --- /dev/null +++ b/rust/kernel/debugfs/callback_adapters.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! Adapters which allow the user to supply a write or read implementation as a value rather +//! than a trait implementation. If provided, it will override the trait implementation. + +use super::{ + Reader, + Writer, // +}; + +use crate::{ + fmt, + prelude::*, + uaccess::UserSliceReader, // +}; + +use core::{ + marker::PhantomData, + ops::Deref, // +}; + +/// # Safety +/// +/// To implement this trait, it must be safe to cast a `&Self` to a `&Inner`. +/// It is intended for use in unstacking adapters out of `FileOps` backings. +pub(crate) unsafe trait Adapter { + type Inner; +} + +/// Adapter to implement `Reader` via a callback with the same representation as `T`. +/// +/// * Layer it on top of `WriterAdapter` if you want to add a custom callback for `write`. +/// * Layer it on top of `NoWriter` to pass through any support present on the underlying type. +/// +/// # Invariants +/// +/// If an instance for `WritableAdapter<_, W>` is constructed, `W` is inhabited. +#[repr(transparent)] +pub(crate) struct WritableAdapter { + inner: D, + _writer: PhantomData, +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for WritableAdapter { + type Inner = D; +} + +impl Writer for WritableAdapter { + fn write(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.write(fmt) + } +} + +impl Reader for WritableAdapter +where + W: Fn(&D::Target, &mut UserSliceReader) -> Result + Send + Sync + 'static, +{ + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + // SAFETY: WritableAdapter<_, W> can only be constructed if W is inhabited + let w: &W = unsafe { materialize_zst() }; + w(self.inner.deref(), reader) + } +} + +/// Adapter to implement `Writer` via a callback with the same representation as `T`. +/// +/// # Invariants +/// +/// If an instance for `FormatAdapter<_, F>` is constructed, `F` is inhabited. +#[repr(transparent)] +pub(crate) struct FormatAdapter { + inner: D, + _formatter: PhantomData, +} + +impl Deref for FormatAdapter { + type Target = D; + fn deref(&self) -> &D { + &self.inner + } +} + +impl Writer for FormatAdapter +where + F: Fn(&D, &mut fmt::Formatter<'_>) -> fmt::Result + 'static, +{ + fn write(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: FormatAdapter<_, F> can only be constructed if F is inhabited + let f: &F = unsafe { materialize_zst() }; + f(&self.inner, fmt) + } +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for FormatAdapter { + type Inner = D; +} + +#[repr(transparent)] +pub(crate) struct NoWriter { + inner: D, +} + +// SAFETY: Stripping off the adapter only removes constraints +unsafe impl Adapter for NoWriter { + type Inner = D; +} + +impl Deref for NoWriter { + type Target = D; + fn deref(&self) -> &D { + &self.inner + } +} + +/// For types with a unique value, produce a static reference to it. +/// +/// # Safety +/// +/// The caller asserts that F is inhabited +unsafe fn materialize_zst() -> &'static F { + const { assert!(core::mem::size_of::() == 0) }; + let zst_dangle: core::ptr::NonNull = core::ptr::NonNull::dangling(); + // SAFETY: While the pointer is dangling, it is a dangling pointer to a ZST, based on the + // assertion above. The type is also inhabited, by the caller's assertion. This means + // we can materialize it. + unsafe { zst_dangle.as_ref() } +} diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs new file mode 100644 index 000000000..46aad6489 --- /dev/null +++ b/rust/kernel/debugfs/entry.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +use crate::{ + debugfs::file_ops::FileOps, + prelude::*, + str::{ + CStr, + CStrExt as _, // + }, + sync::Arc, +}; + +use core::marker::PhantomData; + +/// Owning handle to a DebugFS entry. +/// +/// # Invariants +/// +/// The wrapped pointer will always be `NULL`, an error, or an owned DebugFS `dentry`. +pub(crate) struct Entry<'a> { + entry: *mut bindings::dentry, + // If we were created with an owning parent, this is the keep-alive + _parent: Option>>, + // If we were created with a non-owning parent, this prevents us from outliving it + _phantom: PhantomData<&'a ()>, +} + +// SAFETY: [`Entry`] is just a `dentry` under the hood, which the API promises can be transferred +// between threads. +unsafe impl Send for Entry<'_> {} + +// SAFETY: All the C functions we call on the `dentry` pointer are threadsafe. +unsafe impl Sync for Entry<'_> {} + +impl Entry<'static> { + pub(crate) fn dynamic_dir(name: &CStr, parent: Option>) -> Self { + let parent_ptr = match &parent { + Some(entry) => entry.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent_ptr` is either `NULL` (if `parent` is `None`), or a pointer to a valid + // `dentry` by our invariant. `debugfs_create_dir` handles `NULL` pointers correctly. + let entry = unsafe { bindings::debugfs_create_dir(name.as_char_ptr(), parent_ptr) }; + + Entry { + entry, + _parent: parent, + _phantom: PhantomData, + } + } + + /// # Safety + /// + /// * `data` must outlive the returned `Entry`. + pub(crate) unsafe fn dynamic_file( + name: &CStr, + parent: Arc, + data: &T, + file_ops: &'static FileOps, + ) -> Self { + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent.as_ptr()` is a pointer to a valid `dentry` by invariant. + // * The caller guarantees that `data` will outlive the returned `Entry`. + // * The guarantees on `FileOps` assert the vtable will be compatible with the data we have + // provided. + let entry = unsafe { + bindings::debugfs_create_file_full( + name.as_char_ptr(), + file_ops.mode(), + parent.as_ptr(), + core::ptr::from_ref(data) as *mut c_void, + core::ptr::null(), + &**file_ops, + ) + }; + + Entry { + entry, + _parent: Some(parent), + _phantom: PhantomData, + } + } +} + +impl<'a> Entry<'a> { + pub(crate) fn dir(name: &CStr, parent: Option<&'a Entry<'_>>) -> Self { + let parent_ptr = match &parent { + Some(entry) => entry.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent_ptr` is either `NULL` (if `parent` is `None`), or a pointer to a valid + // `dentry` (because `parent` is a valid reference to an `Entry`). The lifetime `'a` + // ensures that the parent outlives this entry. + let entry = unsafe { bindings::debugfs_create_dir(name.as_char_ptr(), parent_ptr) }; + + Entry { + entry, + _parent: None, + _phantom: PhantomData, + } + } + + pub(crate) fn file( + name: &CStr, + parent: &'a Entry<'_>, + data: &'a T, + file_ops: &FileOps, + ) -> Self { + // SAFETY: The invariants of this function's arguments ensure the safety of this call. + // * `name` is a valid C string by the invariants of `&CStr`. + // * `parent.as_ptr()` is a pointer to a valid `dentry` because we have `&'a Entry`. + // * `data` is a valid pointer to `T` for lifetime `'a`. + // * The returned `Entry` has lifetime `'a`, so it cannot outlive `parent` or `data`. + // * The caller guarantees that `vtable` is compatible with `data`. + // * The guarantees on `FileOps` assert the vtable will be compatible with the data we have + // provided. + let entry = unsafe { + bindings::debugfs_create_file_full( + name.as_char_ptr(), + file_ops.mode(), + parent.as_ptr(), + core::ptr::from_ref(data) as *mut c_void, + core::ptr::null(), + &**file_ops, + ) + }; + + Entry { + entry, + _parent: None, + _phantom: PhantomData, + } + } +} + +impl Entry<'_> { + /// Constructs a placeholder DebugFS [`Entry`]. + pub(crate) fn empty() -> Self { + Self { + entry: core::ptr::null_mut(), + _parent: None, + _phantom: PhantomData, + } + } + + /// Returns the pointer representation of the DebugFS directory. + /// + /// # Guarantees + /// + /// Due to the type invariant, the value returned from this function will always be an error + /// code, `NULL`, or a live DebugFS directory. If it is live, it will remain live at least as + /// long as this entry lives. + pub(crate) fn as_ptr(&self) -> *mut bindings::dentry { + self.entry + } +} + +impl Drop for Entry<'_> { + fn drop(&mut self) { + // SAFETY: `debugfs_remove` can take `NULL`, error values, and legal DebugFS dentries. + // `as_ptr` guarantees that the pointer is of this form. + unsafe { bindings::debugfs_remove(self.as_ptr()) } + } +} diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs new file mode 100644 index 000000000..f15908f71 --- /dev/null +++ b/rust/kernel/debugfs/file_ops.rs @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +use super::{ + BinaryReader, + BinaryWriter, + Reader, + Writer, // +}; + +use crate::{ + debugfs::callback_adapters::Adapter, + fmt, + fs::file, + prelude::*, + seq_file::SeqFile, + seq_print, + uaccess::UserSlice, // +}; + +use core::marker::PhantomData; + +#[cfg(CONFIG_DEBUG_FS)] +use core::ops::Deref; + +/// # Invariant +/// +/// `FileOps` will always contain an `operations` which is safe to use for a file backed +/// off an inode which has a pointer to a `T` in its private data that is safe to convert +/// into a reference. +pub(super) struct FileOps { + #[cfg(CONFIG_DEBUG_FS)] + operations: bindings::file_operations, + #[cfg(CONFIG_DEBUG_FS)] + mode: u16, + _phantom: PhantomData, +} + +impl FileOps { + /// # Safety + /// + /// The caller asserts that the provided `operations` is safe to use for a file whose + /// inode has a pointer to `T` in its private data that is safe to convert into a reference. + const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self { + Self { + #[cfg(CONFIG_DEBUG_FS)] + operations, + #[cfg(CONFIG_DEBUG_FS)] + mode, + _phantom: PhantomData, + } + } + + #[cfg(CONFIG_DEBUG_FS)] + pub(crate) const fn mode(&self) -> u16 { + self.mode + } +} + +impl FileOps { + pub(super) const fn adapt(&self) -> &FileOps { + // SAFETY: `Adapter` asserts that `T` can be legally cast to `T::Inner`. + unsafe { core::mem::transmute(self) } + } +} + +#[cfg(CONFIG_DEBUG_FS)] +impl Deref for FileOps { + type Target = bindings::file_operations; + + fn deref(&self) -> &Self::Target { + &self.operations + } +} + +struct WriterAdapter(T); + +impl<'a, T: Writer> fmt::Display for WriterAdapter<&'a T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.write(f) + } +} + +/// Implements `open` for `file_operations` via `single_open` to fill out a `seq_file`. +/// +/// # Safety +/// +/// * `inode`'s private pointer must point to a value of type `T` which will outlive the `inode` +/// and will not have any unique references alias it during the call. +/// * `file` must point to a live, not-yet-initialized file object. +unsafe extern "C" fn writer_open( + inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_int { + // SAFETY: The caller ensures that `inode` is a valid pointer. + let data = unsafe { (*inode).i_private }; + // SAFETY: + // * `file` is acceptable by caller precondition. + // * `print_act` will be called on a `seq_file` with private data set to the third argument, + // so we meet its safety requirements. + // * The `data` pointer passed in the third argument is a valid `T` pointer that outlives + // this call by caller preconditions. + unsafe { bindings::single_open(file, Some(writer_act::), data) } +} + +/// Prints private data stashed in a seq_file to that seq file. +/// +/// # Safety +/// +/// `seq` must point to a live `seq_file` whose private data is a valid pointer to a `T` which may +/// not have any unique references alias it during the call. +unsafe extern "C" fn writer_act( + seq: *mut bindings::seq_file, + _: *mut c_void, +) -> c_int { + // SAFETY: By caller precondition, this pointer is valid pointer to a `T`, and + // there are not and will not be any unique references until we are done. + let data = unsafe { &*((*seq).private.cast::()) }; + // SAFETY: By caller precondition, `seq_file` points to a live `seq_file`, so we can lift + // it. + let seq_file = unsafe { SeqFile::from_raw(seq) }; + seq_print!(seq_file, "{}", WriterAdapter(data)); + 0 +} + +// Work around lack of generic const items. +pub(crate) trait ReadFile { + const FILE_OPS: FileOps; +} + +impl ReadFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(bindings::seq_read), + llseek: Some(bindings::seq_lseek), + release: Some(bindings::single_release), + open: Some(writer_open::), + ..pin_init::zeroed() + }; + // SAFETY: `operations` is all stock `seq_file` implementations except for `writer_open`. + // `open`'s only requirement beyond what is provided to all open functions is that the + // inode's data pointer must point to a `T` that will outlive it, which matches the + // `FileOps` requirements. + unsafe { FileOps::new(operations, 0o400) } + }; +} + +fn read(data: &T, buf: *const c_char, count: usize) -> isize { + let mut reader = UserSlice::new(UserPtr::from_ptr(buf as *mut c_void), count).reader(); + + if let Err(e) = data.read_from_slice(&mut reader) { + return e.to_errno() as isize; + } + + count as isize +} + +/// # Safety +/// +/// `file` must be a valid pointer to a `file` struct. +/// The `private_data` of the file must contain a valid pointer to a `seq_file` whose +/// `private` data in turn points to a `T` that implements `Reader`. +/// `buf` must be a valid user-space buffer. +pub(crate) unsafe extern "C" fn write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + _ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: The file was opened with `single_open`, which sets `private_data` to a `seq_file`. + let seq = unsafe { &mut *((*file).private_data.cast::()) }; + // SAFETY: By caller precondition, this pointer is live and points to a value of type `T`. + let data = unsafe { &*(seq.private as *const T) }; + read(data, buf, count) +} + +// A trait to get the file operations for a type. +pub(crate) trait ReadWriteFile { + const FILE_OPS: FileOps; +} + +impl ReadWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + open: Some(writer_open::), + read: Some(bindings::seq_read), + write: Some(write::), + llseek: Some(bindings::seq_lseek), + release: Some(bindings::single_release), + ..pin_init::zeroed() + }; + // SAFETY: `operations` is all stock `seq_file` implementations except for `writer_open` + // and `write`. + // `writer_open`'s only requirement beyond what is provided to all open functions is that + // the inode's data pointer must point to a `T` that will outlive it, which matches the + // `FileOps` requirements. + // `write` only requires that the file's private data pointer points to `seq_file` + // which points to a `T` that will outlive it, which matches what `writer_open` + // provides. + unsafe { FileOps::new(operations, 0o600) } + }; +} + +/// # Safety +/// +/// `inode` must be a valid pointer to an `inode` struct. +/// `file` must be a valid pointer to a `file` struct. +unsafe extern "C" fn write_only_open( + inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_int { + // SAFETY: The caller ensures that `inode` and `file` are valid pointers. + unsafe { (*file).private_data = (*inode).i_private }; + 0 +} + +/// # Safety +/// +/// * `file` must be a valid pointer to a `file` struct. +/// * The `private_data` of the file must contain a valid pointer to a `T` that implements +/// `Reader`. +/// * `buf` must be a valid user-space buffer. +pub(crate) unsafe extern "C" fn write_only_write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + _ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: The caller ensures that `file` is a valid pointer and that `private_data` holds a + // valid pointer to `T`. + let data = unsafe { &*((*file).private_data as *const T) }; + read(data, buf, count) +} + +pub(crate) trait WriteFile { + const FILE_OPS: FileOps; +} + +impl WriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + open: Some(write_only_open), + write: Some(write_only_write::), + llseek: Some(bindings::noop_llseek), + ..pin_init::zeroed() + }; + // SAFETY: + // * `write_only_open` populates the file private data with the inode private data + // * `write_only_write`'s only requirement is that the private data of the file point to + // a `T` and be legal to convert to a shared reference, which `write_only_open` + // satisfies. + unsafe { FileOps::new(operations, 0o200) } + }; +} + +extern "C" fn blob_read( + file: *mut bindings::file, + buf: *mut c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut writer = UserSlice::new(UserPtr::from_ptr(buf.cast()), count).writer(); + + let ret = || -> Result { + let written = this.write_to_slice(&mut writer, pos)?; + + Ok(written.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for read only binary files. +pub(crate) trait BinaryReadFile { + const FILE_OPS: FileOps; +} + +impl BinaryReadFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(blob_read::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o400) } + }; +} + +extern "C" fn blob_write( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut reader = UserSlice::new(UserPtr::from_ptr(buf.cast_mut().cast()), count).reader(); + + let ret = || -> Result { + let read = this.read_from_slice(&mut reader, pos)?; + + Ok(read.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for write only binary files. +pub(crate) trait BinaryWriteFile { + const FILE_OPS: FileOps; +} + +impl BinaryWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + write: Some(blob_write::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o200) } + }; +} + +/// Representation of [`FileOps`] for read/write binary files. +pub(crate) trait BinaryReadWriteFile { + const FILE_OPS: FileOps; +} + +impl BinaryReadWriteFile for T { + const FILE_OPS: FileOps = { + let operations = bindings::file_operations { + read: Some(blob_read::), + write: Some(blob_write::), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + ..pin_init::zeroed() + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o600) } + }; +} diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs new file mode 100644 index 000000000..b295f8420 --- /dev/null +++ b/rust/kernel/debugfs/traits.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Google LLC. + +//! Traits for rendering or updating values exported to DebugFS. + +use crate::{ + alloc::Allocator, + fmt, + fs::file, + prelude::*, + sync::{ + atomic::{ + Atomic, + AtomicBasicOps, + AtomicType, + Relaxed, // + }, + Arc, + Mutex, // + }, + uaccess::{ + UserSliceReader, + UserSliceWriter, // + }, +}; + +use core::{ + ops::{ + Deref, + DerefMut, // + }, + str::FromStr, +}; + +use zerocopy::Immutable; + +/// A trait for types that can be written into a string. +/// +/// This works very similarly to `Debug`, and is automatically implemented if `Debug` is +/// implemented for a type. It is also implemented for any writable type inside a `Mutex`. +/// +/// The derived implementation of `Debug` [may +/// change](https://doc.rust-lang.org/std/fmt/trait.Debug.html#stability) +/// between Rust versions, so if stability is key for your use case, please implement `Writer` +/// explicitly instead. +pub trait Writer { + /// Formats the value using the given formatter. + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result; +} + +impl Writer for Mutex { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.lock().write(f) + } +} + +impl Writer for T { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{self:?}") + } +} + +/// Trait for types that can be written out as binary. +pub trait BinaryWriter { + /// Writes the binary form of `self` into `writer`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes written in to `writer`. + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result; +} + +// Base implementation for any `T: Immutable + IntoBytes`. +impl BinaryWriter for T { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + writer.write_slice_file(self.as_bytes(), offset) + } +} + +// Delegate for `Mutex`: Support a `T` with an outer mutex. +impl BinaryWriter for Mutex { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + let guard = self.lock(); + + guard.write_to_slice(writer, offset) + } +} + +// Delegate for `Box`: Support a `Box` with no lock or an inner lock. +impl BinaryWriter for Box +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Pin>`: Support a `Pin>` with no lock or an inner lock. +impl BinaryWriter for Pin> +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Arc`: Support a `Arc` with no lock or an inner lock. +impl BinaryWriter for Arc +where + T: BinaryWriter, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Vec`. +impl BinaryWriter for Vec +where + T: Immutable + IntoBytes, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + writer.write_slice_file(self.as_bytes(), offset) + } +} + +/// A trait for types that can be updated from a user slice. +/// +/// This works similarly to `FromStr`, but operates on a `UserSliceReader` rather than a &str. +/// +/// It is automatically implemented for all atomic integers, or any type that implements `FromStr` +/// wrapped in a `Mutex`. +pub trait Reader { + /// Updates the value from the given user slice. + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result; +} + +impl Reader for Mutex { + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + let mut buf = [0u8; 128]; + if reader.len() > buf.len() { + return Err(EINVAL); + } + let n = reader.len(); + reader.read_slice(&mut buf[..n])?; + + let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; + let val = s.trim().parse::().map_err(|_| EINVAL)?; + *self.lock() = val; + Ok(()) + } +} + +impl Reader for Atomic +where + T::Repr: AtomicBasicOps, +{ + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { + let mut buf = [0u8; 21]; // Enough for a 64-bit number. + if reader.len() > buf.len() { + return Err(EINVAL); + } + let n = reader.len(); + reader.read_slice(&mut buf[..n])?; + + let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; + let val = s.trim().parse::().map_err(|_| EINVAL)?; + self.store(val, Relaxed); + Ok(()) + } +} + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReader`] for interior mutability. +pub trait BinaryReaderMut { + /// Reads the binary form of `self` from `reader`. + /// + /// Same as [`BinaryReader::read_from_slice`], but takes a mutable reference. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result; +} + +// Base implementation for any `T: FromBytes + IntoBytes`. +impl BinaryReaderMut for T { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + reader.read_slice_file(self.as_mut_bytes(), offset) + } +} + +// Delegate for `Box`: Support a `Box` with an outer lock. +impl BinaryReaderMut for Box { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref_mut().read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Vec`: Support a `Vec` with an outer lock. +impl BinaryReaderMut for Vec +where + T: FromBytes + IntoBytes, + A: Allocator, +{ + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + reader.read_slice_file(self.as_mut_bytes(), offset) + } +} + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReaderMut`] for the mutable version. +pub trait BinaryReader { + /// Reads the binary form of `self` from `reader`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result; +} + +// Delegate for `Mutex`: Support a `T` with an outer `Mutex`. +impl BinaryReader for Mutex { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + let mut this = self.lock(); + + this.read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Box`: Support a `Box` with an inner lock. +impl BinaryReader for Box { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Pin>`: Support a `Pin>` with an inner lock. +impl BinaryReader for Pin> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Arc`: Support an `Arc` with an inner lock. +impl BinaryReader for Arc { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result { + self.deref().read_from_slice(reader, offset) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs new file mode 100644 index 000000000..2291d85b6 --- /dev/null +++ b/rust/kernel/device.rs @@ -0,0 +1,952 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic devices that are part of the kernel's driver model. +//! +//! C header: [`include/linux/device.h`](srctree/include/linux/device.h) + +use crate::{ + bindings, + fmt, + prelude::*, + sync::aref::ARef, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::{ + marker::PhantomData, + ptr, // +}; + +pub mod property; + +/// The core representation of a device in the kernel's driver model. +/// +/// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either +/// exist as temporary reference (see also [`Device::from_raw`]), which is only valid within a +/// certain scope or as [`ARef`], owning a dedicated reference count. +/// +/// # Device Types +/// +/// A [`Device`] can represent either a bus device or a class device. +/// +/// ## Bus Devices +/// +/// A bus device is a [`Device`] that is associated with a physical or virtual bus. Examples of +/// buses include PCI, USB, I2C, and SPI. Devices attached to a bus are registered with a specific +/// bus type, which facilitates matching devices with appropriate drivers based on IDs or other +/// identifying information. Bus devices are visible in sysfs under `/sys/bus//devices/`. +/// +/// ## Class Devices +/// +/// A class device is a [`Device`] that is associated with a logical category of functionality +/// rather than a physical bus. Examples of classes include block devices, network interfaces, sound +/// cards, and input devices. Class devices are grouped under a common class and exposed to +/// userspace via entries in `/sys/class//`. +/// +/// # Device Context +/// +/// [`Device`] references are generic over a [`DeviceContext`], which represents the type state of +/// a [`Device`]. +/// +/// As the name indicates, this type state represents the context of the scope the [`Device`] +/// reference is valid in. For instance, the [`Bound`] context guarantees that the [`Device`] is +/// bound to a driver for the entire duration of the existence of a [`Device`] reference. +/// +/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`], [`CoreInternal`] and +/// [`BoundInternal`]. +/// +/// Unless selected otherwise [`Device`] defaults to the [`Normal`] [`DeviceContext`], which by +/// itself has no additional requirements. +/// +/// It is always up to the caller of [`Device::from_raw`] to select the correct [`DeviceContext`] +/// type for the corresponding scope the [`Device`] reference is created in. +/// +/// All [`DeviceContext`] types other than [`Normal`] are intended to be used with +/// [bus devices](#bus-devices) only. +/// +/// # Implementing Bus Devices +/// +/// This section provides a guideline to implement bus specific devices, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`] +/// +/// A bus specific device should be defined as follows. +/// +/// ```ignore +/// #[repr(transparent)] +/// pub struct Device( +/// Opaque, +/// PhantomData, +/// ); +/// ``` +/// +/// Since devices are reference counted, [`AlwaysRefCounted`] should be implemented for `Device` +/// (i.e. `Device`). Note that [`AlwaysRefCounted`] must not be implemented for any other +/// [`DeviceContext`], since all other device context types are only valid within a certain scope. +/// +/// In order to be able to implement the [`DeviceContext`] dereference hierarchy, bus device +/// implementations should call the [`impl_device_context_deref`] macro as shown below. +/// +/// ```ignore +/// // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s +/// // generic argument. +/// kernel::impl_device_context_deref!(unsafe { Device }); +/// ``` +/// +/// In order to convert from a any [`Device`] to [`ARef`], bus devices can implement +/// the following macro call. +/// +/// ```ignore +/// kernel::impl_device_context_into_aref!(Device); +/// ``` +/// +/// Bus devices should also implement the following [`AsRef`] implementation, such that users can +/// easily derive a generic [`Device`] reference. +/// +/// ```ignore +/// impl AsRef> for Device { +/// fn as_ref(&self) -> &device::Device { +/// ... +/// } +/// } +/// ``` +/// +/// # Implementing Class Devices +/// +/// Class device implementations require less infrastructure and depend slightly more on the +/// specific subsystem. +/// +/// An example implementation for a class device could look like this. +/// +/// ```ignore +/// #[repr(C)] +/// pub struct Device { +/// dev: Opaque, +/// data: T::Data, +/// } +/// ``` +/// +/// This class device uses the sub-classing pattern to embed the driver's private data within the +/// allocation of the class device. For this to be possible the class device is generic over the +/// class specific `Driver` trait implementation. +/// +/// Just like any device, class devices are reference counted and should hence implement +/// [`AlwaysRefCounted`] for `Device`. +/// +/// Class devices should also implement the following [`AsRef`] implementation, such that users can +/// easily derive a generic [`Device`] reference. +/// +/// ```ignore +/// impl AsRef for Device { +/// fn as_ref(&self) -> &device::Device { +/// ... +/// } +/// } +/// ``` +/// +/// An example for a class device implementation is +#[cfg_attr(CONFIG_DRM = "y", doc = "[`drm::Device`](kernel::drm::Device).")] +#[cfg_attr(not(CONFIG_DRM = "y"), doc = "`drm::Device`.")] +/// +/// # Invariants +/// +/// A `Device` instance represents a valid `struct device` created by the C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to `get_device` ensures +/// that the allocation remains valid at least until the matching call to `put_device`. +/// +/// `bindings::device::release` is valid to be called from any thread, hence `ARef` can be +/// dropped from any thread. +/// +/// [`AlwaysRefCounted`]: kernel::sync::aref::AlwaysRefCounted +/// [`impl_device_context_deref`]: kernel::impl_device_context_deref +/// [`platform::Device`]: kernel::platform::Device +#[repr(transparent)] +pub struct Device(Opaque, PhantomData); + +impl Device { + /// Creates a new reference-counted abstraction instance of an existing `struct device` pointer. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to + /// can't drop to zero, for the duration of this function call. + /// + /// It must also be ensured that `bindings::device::release` can be called from any thread. + /// While not officially documented, this should be the case for any `struct device`. + pub unsafe fn get_device(ptr: *mut bindings::device) -> ARef { + // SAFETY: By the safety requirements ptr is valid + unsafe { Self::from_raw(ptr) }.into() + } + + /// Convert a [`&Device`](Device) into a [`&Device`](Device). + /// + /// # Safety + /// + /// The caller is responsible to ensure that the returned [`&Device`](Device) + /// only lives as long as it can be guaranteed that the [`Device`] is actually bound. + pub unsafe fn as_bound(&self) -> &Device { + let ptr = core::ptr::from_ref(self); + + // CAST: By the safety requirements the caller is responsible to guarantee that the + // returned reference only lives as long as the device is actually bound. + let ptr = ptr.cast(); + + // SAFETY: + // - `ptr` comes from `from_ref(self)` above, hence it's guaranteed to be valid. + // - Any valid `Device` pointer is also a valid pointer for `Device`. + unsafe { &*ptr } + } +} + +impl<'a> Device> { + /// Store a pointer to the bound driver's private data. + pub fn set_drvdata(&self, data: impl PinInit) -> Result { + let data = KBox::pin_init(data, GFP_KERNEL)?; + + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) }; + + Ok(()) + } + + /// Take ownership of the private data stored in this [`Device`]. + /// + /// # Safety + /// + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) }; + + if ptr.is_null() { + return None; + } + + // SAFETY: + // - If `ptr` is not NULL, it comes from a previous call to `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + Some(unsafe { Pin::>::from_foreign(ptr.cast()) }) + } +} + +impl Device { + /// Borrow the driver's private data bound to this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before the + /// device is fully unbound. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_borrow(&self) -> Pin<&T> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { Pin::>::borrow(ptr.cast()) } + } +} + +impl Device { + /// Obtain the raw `struct device *`. + pub(crate) fn as_raw(&self) -> *mut bindings::device { + self.0.get() + } + + /// Returns a reference to the parent device, if any. + #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] + pub(crate) fn parent(&self) -> Option<&Device> { + // SAFETY: + // - By the type invariant `self.as_raw()` is always valid. + // - The parent device is only ever set at device creation. + let parent = unsafe { (*self.as_raw()).parent }; + + if parent.is_null() { + None + } else { + // SAFETY: + // - Since `parent` is not NULL, it must be a valid pointer to a `struct device`. + // - `parent` is valid for the lifetime of `self`, since a `struct device` holds a + // reference count of its parent. + Some(unsafe { Device::from_raw(parent) }) + } + } + + /// Convert a raw C `struct device` pointer to a `&'a Device`. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to + /// can't drop to zero, for the duration of this function call and the entire duration when the + /// returned reference exists. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::device) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + unsafe { &*ptr.cast() } + } + + /// Prints an emergency-level message (level 0) prefixed with device information. + /// + /// More details are available from [`dev_emerg`]. + /// + /// [`dev_emerg`]: crate::dev_emerg + pub fn pr_emerg(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_EMERG, args) }; + } + + /// Prints an alert-level message (level 1) prefixed with device information. + /// + /// More details are available from [`dev_alert`]. + /// + /// [`dev_alert`]: crate::dev_alert + pub fn pr_alert(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ALERT, args) }; + } + + /// Prints a critical-level message (level 2) prefixed with device information. + /// + /// More details are available from [`dev_crit`]. + /// + /// [`dev_crit`]: crate::dev_crit + pub fn pr_crit(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_CRIT, args) }; + } + + /// Prints an error-level message (level 3) prefixed with device information. + /// + /// More details are available from [`dev_err`]. + /// + /// [`dev_err`]: crate::dev_err + pub fn pr_err(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ERR, args) }; + } + + /// Prints a warning-level message (level 4) prefixed with device information. + /// + /// More details are available from [`dev_warn`]. + /// + /// [`dev_warn`]: crate::dev_warn + pub fn pr_warn(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_WARNING, args) }; + } + + /// Prints a notice-level message (level 5) prefixed with device information. + /// + /// More details are available from [`dev_notice`]. + /// + /// [`dev_notice`]: crate::dev_notice + pub fn pr_notice(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_NOTICE, args) }; + } + + /// Prints an info-level message (level 6) prefixed with device information. + /// + /// More details are available from [`dev_info`]. + /// + /// [`dev_info`]: crate::dev_info + pub fn pr_info(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_INFO, args) }; + } + + /// Prints a debug-level message (level 7) prefixed with device information. + /// + /// More details are available from [`dev_dbg`]. + /// + /// [`dev_dbg`]: crate::dev_dbg + pub fn pr_dbg(&self, args: fmt::Arguments<'_>) { + if cfg!(debug_assertions) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_DEBUG, args) }; + } + } + + /// Prints the provided message to the console. + /// + /// # Safety + /// + /// Callers must ensure that `klevel` is null-terminated; in particular, one of the + /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. + #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] + unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated and one of the kernel constants. `self.as_raw` + // is valid because `self` is valid. The "%pA" format string expects a pointer to + // `fmt::Arguments`, which is what we're passing as the last argument. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_dev_printk( + klevel.as_ptr().cast::(), + self.as_raw(), + c"%pA".as_char_ptr(), + core::ptr::from_ref(&msg).cast::(), + ) + }; + } + + /// Obtain the [`FwNode`](property::FwNode) corresponding to this [`Device`]. + pub fn fwnode(&self) -> Option<&property::FwNode> { + // SAFETY: `self` is valid. + let fwnode_handle = unsafe { bindings::__dev_fwnode(self.as_raw()) }; + if fwnode_handle.is_null() { + return None; + } + // SAFETY: `fwnode_handle` is valid. Its lifetime is tied to `&self`. We + // return a reference instead of an `ARef` because `dev_fwnode()` + // doesn't increment the refcount. It is safe to cast from a + // `struct fwnode_handle*` to a `*const FwNode` because `FwNode` is + // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. + Some(unsafe { &*fwnode_handle.cast() }) + } + + /// Returns the name of the device. + /// + /// This is the kobject name of the device, or its initial name if the kobject is not yet + /// available. + #[inline] + pub fn name(&self) -> &CStr { + // SAFETY: By its type invariant `self.as_raw()` is a valid pointer to a `struct device`. + // The returned string is valid for the lifetime of the device. + unsafe { CStr::from_char_ptr(bindings::dev_name(self.as_raw())) } + } +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(obj.cast().as_ptr()) } + } +} + +// SAFETY: As by the type invariant `Device` can be sent to any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all immutable methods are protected by the +// synchronization in `struct device`. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct device` is the same; `Bound` is a +// zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Marker trait for the context or scope of a bus specific device. +/// +/// [`DeviceContext`] is a marker trait for types representing the context of a bus specific +/// [`Device`]. +/// +/// The specific device context types are: [`CoreInternal`], [`Core`], [`BoundInternal`], [`Bound`] +/// and [`Normal`]. +/// +/// [`DeviceContext`] types are hierarchical, which means that there is a strict hierarchy that +/// defines which [`DeviceContext`] type can be derived from another. For instance, any +/// [`Device`] can dereference to a [`Device`]. +/// +/// The following enumeration illustrates the dereference hierarchy of [`DeviceContext`] types. +/// +/// - [`CoreInternal`] => [`Core`] => [`Bound`] => [`Normal`] +/// - [`BoundInternal`] => [`Bound`] => [`Normal`] +/// +/// Both [`CoreInternal`] and [`BoundInternal`] implement the [`InternalBoundContext`] trait, +/// which provides access to internal bus abstraction methods on [`Device`] that are not available +/// to drivers. +/// +/// Bus devices can automatically implement the dereference hierarchy by using +/// [`impl_device_context_deref`]. +/// +/// Note that the guarantee for a [`Device`] reference to have a certain [`DeviceContext`] comes +/// from the specific scope the [`Device`] reference is valid in. +/// +/// [`impl_device_context_deref`]: kernel::impl_device_context_deref +pub trait DeviceContext: private::Sealed {} + +/// The [`Normal`] context is the default [`DeviceContext`] of any [`Device`]. +/// +/// The normal context does not indicate any specific context. Any `Device` is also a valid +/// [`Device`]. It is the only [`DeviceContext`] for which it is valid to implement +/// [`AlwaysRefCounted`] for. +/// +/// [`AlwaysRefCounted`]: kernel::sync::aref::AlwaysRefCounted +pub struct Normal; + +/// The [`Core`] context is the context of a bus specific device when it appears as argument of +/// any bus specific callback, such as `probe()`. +/// +/// The core context indicates that the [`Device`] reference's scope is limited to the bus +/// callback it appears in. It is intended to be used for synchronization purposes. Bus device +/// implementations can implement methods for [`Device`], such that they can only be called +/// from bus callbacks. +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData &'a ()>); + +/// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal core context is intended to be used in exactly the same way as the [`Core`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context mainly exists to share generic [`Device`] infrastructure that should only be called +/// from bus callbacks with bus abstractions, but without making them accessible for drivers. +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData &'a ()>); + +/// Semantically the same as [`Bound`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal bound context is intended to be used in exactly the same way as the [`Bound`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context exists for cases where the bus abstraction needs access to internal device +/// infrastructure (such as [`Device::drvdata_borrow`]), where [`CoreInternal`] would not be +/// justified. +pub struct BoundInternal; + +/// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to +/// be bound to a driver. +/// +/// The bound context indicates that for the entire duration of the lifetime of a [`Device`] +/// reference, the [`Device`] is guaranteed to be bound to a driver. +/// +/// Some APIs, such as [`dma::Coherent`] or [`Devres`] rely on the [`Device`] to be bound, +/// which can be proven with the [`Bound`] device context. +/// +/// Any abstraction that can guarantee a scope where the corresponding bus device is bound, should +/// provide a [`Device`] reference to its users for this scope. This allows users to benefit +/// from optimizations for accessing device resources, see also [`Devres::access`]. +/// +/// [`Devres`]: kernel::devres::Devres +/// [`Devres::access`]: kernel::devres::Devres::access +/// [`dma::Coherent`]: kernel::dma::Coherent +pub struct Bound; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Bound {} + impl Sealed for super::BoundInternal {} + impl<'a> Sealed for super::Core<'a> {} + impl<'a> Sealed for super::CoreInternal<'a> {} + impl Sealed for super::Normal {} +} + +impl DeviceContext for Bound {} +impl DeviceContext for BoundInternal {} +impl<'a> DeviceContext for Core<'a> {} +impl<'a> DeviceContext for CoreInternal<'a> {} +impl DeviceContext for Normal {} + +/// Marker trait for [`DeviceContext`] types that have internal bound-level access. +/// +/// This trait is implemented by [`CoreInternal`] and [`BoundInternal`], allowing methods that +/// require internal bus abstraction access to a bound device to be generic over both contexts. +/// +/// Methods bounded by this trait are available to bus abstractions but not to drivers. +pub trait InternalBoundContext: DeviceContext {} +impl<'a> InternalBoundContext for CoreInternal<'a> {} +impl InternalBoundContext for BoundInternal {} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &Device { + self + } +} + +/// Convert device references to bus device references. +/// +/// Bus devices can implement this trait to allow abstractions to provide the bus device in +/// class device callbacks. +/// +/// This must not be used by drivers and is intended for bus and class device abstractions only. +/// +/// # Safety +/// +/// `AsBusDevice::OFFSET` must be the offset of the embedded base `struct device` field within a +/// bus device structure. +pub unsafe trait AsBusDevice: AsRef> { + /// The relative offset to the device field. + /// + /// Use `offset_of!(bindings, field)` macro to avoid breakage. + const OFFSET: usize; + + /// Convert a reference to [`Device`] into `Self`. + /// + /// # Safety + /// + /// `dev` must be contained in `Self`. + unsafe fn from_device(dev: &Device) -> &Self + where + Self: Sized, + { + let raw = dev.as_raw(); + // SAFETY: `raw - Self::OFFSET` is guaranteed by the safety requirements + // to be a valid pointer to `Self`. + unsafe { &*raw.byte_sub(Self::OFFSET).cast::() } + } +} + +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_deref { + (unsafe { $device:ident, <$lt:lifetime> $src:ty => $dst:ty }) => { + impl<$lt> ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; + (unsafe { $device:ident, $src:ty => $dst:ty }) => { + impl ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; +} + +/// Implement [`core::ops::Deref`] traits for allowed [`DeviceContext`] conversions of a (bus +/// specific) device. +/// +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[macro_export] +macro_rules! impl_device_context_deref { + (unsafe { $device:ident }) => { + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + <'a> $crate::device::CoreInternal<'a> => $crate::device::Core<'a> + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + <'a> $crate::device::Core<'a> => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::BoundInternal => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::Bound => $crate::device::Normal + }); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_into_aref { + (<$lt:lifetime> $src:ty, $device:tt) => { + impl<$lt> ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; + ($src:ty, $device:tt) => { + impl ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; +} + +/// Implement [`core::convert::From`], such that all `&Device` can be converted to an +/// `ARef`. +#[macro_export] +macro_rules! impl_device_context_into_aref { + ($device:tt) => { + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::CoreInternal<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::Core<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!($crate::device::BoundInternal, $device); + ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! dev_printk { + ($method:ident, $dev:expr, $($f:tt)*) => { + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) + } +} + +/// Prints an emergency-level message (level 0) prefixed with device information. +/// +/// This level should be used if the system is unusable. +/// +/// Equivalent to the kernel's `dev_emerg` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_emerg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_emerg { + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) } +} + +/// Prints an alert-level message (level 1) prefixed with device information. +/// +/// This level should be used if action must be taken immediately. +/// +/// Equivalent to the kernel's `dev_alert` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_alert!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_alert { + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) } +} + +/// Prints a critical-level message (level 2) prefixed with device information. +/// +/// This level should be used in critical conditions. +/// +/// Equivalent to the kernel's `dev_crit` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_crit!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_crit { + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) } +} + +/// Prints an error-level message (level 3) prefixed with device information. +/// +/// This level should be used in error conditions. +/// +/// Equivalent to the kernel's `dev_err` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_err!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_err { + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) } +} + +/// Prints a warning-level message (level 4) prefixed with device information. +/// +/// This level should be used in warning conditions. +/// +/// Equivalent to the kernel's `dev_warn` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_warn!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_warn { + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) } +} + +/// Prints a notice-level message (level 5) prefixed with device information. +/// +/// This level should be used in normal but significant conditions. +/// +/// Equivalent to the kernel's `dev_notice` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_notice!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_notice { + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) } +} + +/// Prints an info-level message (level 6) prefixed with device information. +/// +/// This level should be used for informational messages. +/// +/// Equivalent to the kernel's `dev_info` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_info!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_info { + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) } +} + +/// Prints a debug-level message (level 7) prefixed with device information. +/// +/// This level should be used for debug messages. +/// +/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`std::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_dbg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_dbg { + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) } +} diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs new file mode 100644 index 000000000..5aead835f --- /dev/null +++ b/rust/kernel/device/property.rs @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Unified device property interface. +//! +//! C header: [`include/linux/property.h`](srctree/include/linux/property.h) + +use core::{mem::MaybeUninit, ptr}; + +use super::private::Sealed; +use crate::{ + alloc::KVec, + bindings, + error::{to_result, Result}, + fmt, + prelude::*, + str::{CStr, CString}, + sync::aref::ARef, + types::Opaque, +}; + +/// A reference-counted fwnode_handle. +/// +/// This structure represents the Rust abstraction for a +/// C `struct fwnode_handle`. This implementation abstracts the usage of an +/// already existing C `struct fwnode_handle` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A `FwNode` instance represents a valid `struct fwnode_handle` created by the +/// C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to +/// `fwnode_handle_get` ensures that the allocation remains valid at least until +/// the matching call to `fwnode_handle_put`. +#[repr(transparent)] +pub struct FwNode(Opaque); + +impl FwNode { + /// # Safety + /// + /// Callers must ensure that: + /// - The reference count was incremented at least once. + /// - They relinquish that increment. That is, if there is only one + /// increment, callers must not use the underlying object anymore -- it is + /// only safe to do so via the newly created `ARef`. + unsafe fn from_raw(raw: *mut bindings::fwnode_handle) -> ARef { + // SAFETY: As per the safety requirements of this function: + // - `NonNull::new_unchecked`: + // - `raw` is not null. + // - `ARef::from_raw`: + // - `raw` has an incremented refcount. + // - that increment is relinquished, i.e. it won't be decremented + // elsewhere. + // CAST: It is safe to cast from a `*mut fwnode_handle` to + // `*mut FwNode`, because `FwNode` is defined as a + // `#[repr(transparent)]` wrapper around `fwnode_handle`. + unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(raw.cast())) } + } + + /// Obtain the raw `struct fwnode_handle *`. + pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + self.0.get() + } + + /// Returns `true` if `&self` is an OF node, `false` otherwise. + pub fn is_of_node(&self) -> bool { + // SAFETY: The type invariant of `Self` guarantees that `self.as_raw() is a pointer to a + // valid `struct fwnode_handle`. + unsafe { bindings::is_of_node(self.as_raw()) } + } + + /// Returns an object that implements [`Display`](fmt::Display) for + /// printing the name of a node. + /// + /// This is an alternative to the default `Display` implementation, which + /// prints the full path. + pub fn display_name(&self) -> impl fmt::Display + '_ { + struct FwNodeDisplayName<'a>(&'a FwNode); + + impl fmt::Display for FwNodeDisplayName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: `self` is valid by its type invariant. + let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) }; + if name.is_null() { + return Ok(()); + } + // SAFETY: + // - `fwnode_get_name` returns null or a valid C string. + // - `name` was checked to be non-null. + let name = unsafe { CStr::from_char_ptr(name) }; + fmt::Display::fmt(name, f) + } + } + + FwNodeDisplayName(self) + } + + /// Checks if property is present or not. + pub fn property_present(&self, name: &CStr) -> bool { + // SAFETY: By the invariant of `CStr`, `name` is null-terminated. + unsafe { bindings::fwnode_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + } + + /// Returns firmware property `name` boolean value. + pub fn property_read_bool(&self, name: &CStr) -> bool { + // SAFETY: + // - `name` is non-null and null-terminated. + // - `self.as_raw()` is valid because `self` is valid. + unsafe { bindings::fwnode_property_read_bool(self.as_raw(), name.as_char_ptr()) } + } + + /// Returns the index of matching string `match_str` for firmware string + /// property `name`. + pub fn property_match_string(&self, name: &CStr, match_str: &CStr) -> Result { + // SAFETY: + // - `name` and `match_str` are non-null and null-terminated. + // - `self.as_raw` is valid because `self` is valid. + let ret = unsafe { + bindings::fwnode_property_match_string( + self.as_raw(), + name.as_char_ptr(), + match_str.as_char_ptr(), + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + + /// Returns firmware property `name` integer array values in a [`KVec`]. + pub fn property_read_array_vec<'fwnode, 'name, T: PropertyInt>( + &'fwnode self, + name: &'name CStr, + len: usize, + ) -> Result>> { + let mut val: KVec = KVec::with_capacity(len, GFP_KERNEL)?; + + let res = T::read_array_from_fwnode_property(self, name, val.spare_capacity_mut()); + let res = match res { + Ok(_) => { + // SAFETY: + // - `len` is equal to `val.capacity - val.len`, because + // `val.capacity` is `len` and `val.len` is zero. + // - All elements within the interval [`0`, `len`) were initialized + // by `read_array_from_fwnode_property`. + unsafe { val.inc_len(len) } + Ok(val) + } + Err(e) => Err(e), + }; + Ok(PropertyGuard { + inner: res, + fwnode: self, + name, + }) + } + + /// Returns integer array length for firmware property `name`. + pub fn property_count_elem(&self, name: &CStr) -> Result { + T::read_array_len_from_fwnode_property(self, name) + } + + /// Returns the value of firmware property `name`. + /// + /// This method is generic over the type of value to read. The types that + /// can be read are strings, integers and arrays of integers. + /// + /// Reading a [`KVec`] of integers is done with the separate + /// method [`Self::property_read_array_vec`], because it takes an + /// additional `len` argument. + /// + /// Reading a boolean is done with the separate method + /// [`Self::property_read_bool`], because this operation is infallible. + /// + /// For more precise documentation about what types can be read, see + /// the [implementors of Property][Property#implementors] and [its + /// implementations on foreign types][Property#foreign-impls]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{device::{Device, property::FwNode}, str::CString}; + /// fn examples(dev: &Device) -> Result { + /// let fwnode = dev.fwnode().ok_or(ENOENT)?; + /// let b: u32 = fwnode.property_read(c"some-number").required_by(dev)?; + /// if let Some(s) = fwnode.property_read::(c"some-str").optional() { + /// // ... + /// } + /// Ok(()) + /// } + /// ``` + pub fn property_read<'fwnode, 'name, T: Property>( + &'fwnode self, + name: &'name CStr, + ) -> PropertyGuard<'fwnode, 'name, T> { + PropertyGuard { + inner: T::read_from_fwnode_property(self, name), + fwnode: self, + name, + } + } + + /// Returns first matching named child node handle. + pub fn get_child_by_name(&self, name: &CStr) -> Option> { + // SAFETY: `self` and `name` are valid by their type invariants. + let child = + unsafe { bindings::fwnode_get_named_child_node(self.as_raw(), name.as_char_ptr()) }; + if child.is_null() { + return None; + } + // SAFETY: + // - `fwnode_get_named_child_node` returns a pointer with its refcount + // incremented. + // - That increment is relinquished, i.e. the underlying object is not + // used anymore except via the newly created `ARef`. + Some(unsafe { Self::from_raw(child) }) + } + + /// Returns an iterator over a node's children. + pub fn children<'a>(&'a self) -> impl Iterator> + 'a { + let mut prev: Option> = None; + + core::iter::from_fn(move || { + let prev_ptr = match prev.take() { + None => ptr::null_mut(), + Some(prev) => { + // We will pass `prev` to `fwnode_get_next_child_node`, + // which decrements its refcount, so we use + // `ARef::into_raw` to avoid decrementing the refcount + // twice. + let prev = ARef::into_raw(prev); + prev.as_ptr().cast() + } + }; + // SAFETY: + // - `self.as_raw()` is valid by its type invariant. + // - `prev_ptr` may be null, which is allowed and corresponds to + // getting the first child. Otherwise, `prev_ptr` is valid, as it + // is the stored return value from the previous invocation. + // - `prev_ptr` has its refount incremented. + // - The increment of `prev_ptr` is relinquished, i.e. the + // underlying object won't be used anymore. + let next = unsafe { bindings::fwnode_get_next_child_node(self.as_raw(), prev_ptr) }; + if next.is_null() { + return None; + } + // SAFETY: + // - `next` is valid because `fwnode_get_next_child_node` returns a + // pointer with its refcount incremented. + // - That increment is relinquished, i.e. the underlying object + // won't be used anymore, except via the newly created + // `ARef`. + let next = unsafe { FwNode::from_raw(next) }; + prev = Some(next.clone()); + Some(next) + }) + } + + /// Finds a reference with arguments. + pub fn property_get_reference_args( + &self, + prop: &CStr, + nargs: NArgs<'_>, + index: u32, + ) -> Result { + let mut out_args = FwNodeReferenceArgs::default(); + + let (nargs_prop, nargs) = match nargs { + NArgs::Prop(nargs_prop) => (nargs_prop.as_char_ptr(), 0), + NArgs::N(nargs) => (ptr::null(), nargs), + }; + + // SAFETY: + // - `self.0.get()` is valid. + // - `prop.as_char_ptr()` is valid and zero-terminated. + // - `nargs_prop` is valid and zero-terminated if `nargs` + // is zero, otherwise it is allowed to be a null-pointer. + // - The function upholds the type invariants of `out_args`, + // namely: + // - It may fill the field `fwnode` with a valid pointer, + // in which case its refcount is incremented. + // - It may modify the field `nargs`, in which case it + // initializes at least as many elements in `args`. + let ret = unsafe { + bindings::fwnode_property_get_reference_args( + self.0.get(), + prop.as_char_ptr(), + nargs_prop, + nargs, + index, + &mut out_args.0, + ) + }; + to_result(ret)?; + + Ok(out_args) + } +} + +/// The number of arguments to request [`FwNodeReferenceArgs`]. +pub enum NArgs<'a> { + /// The name of the property of the reference indicating the number of + /// arguments. + Prop(&'a CStr), + /// The known number of arguments. + N(u32), +} + +/// The return value of [`FwNode::property_get_reference_args`]. +/// +/// This structure represents the Rust abstraction for a C +/// `struct fwnode_reference_args` which was initialized by the C side. +/// +/// # Invariants +/// +/// If the field `fwnode` is valid, it owns an increment of its refcount. +/// +/// The field `args` contains at least as many initialized elements as indicated +/// by the field `nargs`. +#[repr(transparent)] +#[derive(Default)] +pub struct FwNodeReferenceArgs(bindings::fwnode_reference_args); + +impl Drop for FwNodeReferenceArgs { + fn drop(&mut self) { + if !self.0.fwnode.is_null() { + // SAFETY: + // - By the type invariants of `FwNodeReferenceArgs`, its field + // `fwnode` owns an increment of its refcount. + // - That increment is relinquished. The underlying object won't be + // used anymore because we are dropping it. + let _ = unsafe { FwNode::from_raw(self.0.fwnode) }; + } + } +} + +impl FwNodeReferenceArgs { + /// Returns the slice of reference arguments. + pub fn as_slice(&self) -> &[u64] { + // SAFETY: As per the safety invariant of `FwNodeReferenceArgs`, `nargs` + // is the minimum number of elements in `args` that is valid. + unsafe { core::slice::from_raw_parts(self.0.args.as_ptr(), self.0.nargs as usize) } + } + + /// Returns the number of reference arguments. + pub fn len(&self) -> usize { + self.0.nargs as usize + } + + /// Returns `true` if there are no reference arguments. + pub fn is_empty(&self) -> bool { + self.0.nargs == 0 + } +} + +impl fmt::Debug for FwNodeReferenceArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +// SAFETY: Instances of `FwNode` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for FwNode { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the + // refcount is non-zero. + unsafe { bindings::fwnode_handle_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is + // non-zero. + unsafe { bindings::fwnode_handle_put(obj.cast().as_ptr()) } + } +} + +enum Node<'a> { + Borrowed(&'a FwNode), + Owned(ARef), +} + +impl fmt::Display for FwNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The logic here is the same as the one in lib/vsprintf.c + // (fwnode_full_name_string). + + // SAFETY: `self.as_raw()` is valid by its type invariant. + let num_parents = unsafe { bindings::fwnode_count_parents(self.as_raw()) }; + + for depth in (0..=num_parents).rev() { + let fwnode = if depth == 0 { + Node::Borrowed(self) + } else { + // SAFETY: `self.as_raw()` is valid. + let ptr = unsafe { bindings::fwnode_get_nth_parent(self.as_raw(), depth) }; + // SAFETY: + // - The depth passed to `fwnode_get_nth_parent` is + // within the valid range, so the returned pointer is + // not null. + // - The reference count was incremented by + // `fwnode_get_nth_parent`. + // - That increment is relinquished to + // `FwNode::from_raw`. + Node::Owned(unsafe { FwNode::from_raw(ptr) }) + }; + // Take a reference to the owned or borrowed `FwNode`. + let fwnode: &FwNode = match &fwnode { + Node::Borrowed(f) => f, + Node::Owned(f) => f, + }; + + // SAFETY: `fwnode` is valid by its type invariant. + let prefix = unsafe { bindings::fwnode_get_name_prefix(fwnode.as_raw()) }; + if !prefix.is_null() { + // SAFETY: `fwnode_get_name_prefix` returns null or a + // valid C string. + let prefix = unsafe { CStr::from_char_ptr(prefix) }; + fmt::Display::fmt(prefix, f)?; + } + fmt::Display::fmt(&fwnode.display_name(), f)?; + } + + Ok(()) + } +} + +/// Implemented for types that can be read as properties. +/// +/// This is implemented for strings, integers and arrays of integers. It's used +/// to make [`FwNode::property_read`] generic over the type of property being +/// read. There are also two dedicated methods to read other types, because they +/// require more specialized function signatures: +/// - [`property_read_bool`](FwNode::property_read_bool) +/// - [`property_read_array_vec`](FwNode::property_read_array_vec) +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait Property: Sized + Sealed { + /// Used to make [`FwNode::property_read`] generic. + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result; +} + +impl Sealed for CString {} + +impl Property for CString { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let mut str: *mut u8 = ptr::null_mut(); + let pstr: *mut _ = &mut str; + + // SAFETY: + // - `name` is non-null and null-terminated. + // - `fwnode.as_raw` is valid because `fwnode` is valid. + let ret = unsafe { + bindings::fwnode_property_read_string(fwnode.as_raw(), name.as_char_ptr(), pstr.cast()) + }; + to_result(ret)?; + + // SAFETY: + // - `pstr` is a valid pointer to a NUL-terminated C string. + // - It is valid for at least as long as `fwnode`, but it's only used + // within the current function. + // - The memory it points to is not mutated during that time. + let str = unsafe { CStr::from_char_ptr(*pstr) }; + Ok(str.try_into()?) + } +} + +/// Implemented for all integers that can be read as properties. +/// +/// This helper trait is needed on top of the existing [`Property`] +/// trait to associate the integer types of various sizes with their +/// corresponding `fwnode_property_read_*_array` functions. +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait PropertyInt: Copy + Sealed { + /// Reads a property array. + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit], + ) -> Result<&'a mut [Self]>; + + /// Reads the length of a property array. + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result; +} +// This macro generates implementations of the traits `Property` and +// `PropertyInt` for integers of various sizes. Its input is a list +// of pairs separated by commas. The first element of the pair is the +// type of the integer, the second one is the name of its corresponding +// `fwnode_property_read_*_array` function. +macro_rules! impl_property_for_int { + ($($int:ty: $f:ident),* $(,)?) => { $( + impl Sealed for $int {} + impl Sealed for [$int; N] {} + + impl PropertyInt for $int { + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit], + ) -> Result<&'a mut [Self]> { + // SAFETY: + // - `fwnode`, `name` and `out` are all valid by their type + // invariants. + // - `out.len()` is a valid bound for the memory pointed to by + // `out.as_mut_ptr()`. + // CAST: It's ok to cast from `*mut MaybeUninit<$int>` to a + // `*mut $int` because they have the same memory layout. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + out.as_mut_ptr().cast(), + out.len(), + ) + }; + to_result(ret)?; + // SAFETY: Transmuting from `&'a mut [MaybeUninit]` to + // `&'a mut [Self]` is sound, because the previous call to a + // `fwnode_property_read_*_array` function (which didn't fail) + // fully initialized the slice. + Ok(unsafe { core::mem::transmute::<&mut [MaybeUninit], &mut [Self]>(out) }) + } + + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + // SAFETY: + // - `fwnode` and `name` are valid by their type invariants. + // - It's ok to pass a null pointer to the + // `fwnode_property_read_*_array` functions if `nval` is zero. + // This will return the length of the array. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + ptr::null_mut(), + 0, + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + } + + impl Property for $int { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let val: [_; 1] = <[$int; 1]>::read_from_fwnode_property(fwnode, name)?; + Ok(val[0]) + } + } + + impl Property for [$int; N] { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result { + let mut val: [MaybeUninit<$int>; N] = [const { MaybeUninit::uninit() }; N]; + + <$int>::read_array_from_fwnode_property(fwnode, name, &mut val)?; + + // SAFETY: `val` is always initialized when + // `fwnode_property_read_*_array` is successful. + Ok(val.map(|v| unsafe { v.assume_init() })) + } + } + )* }; +} +impl_property_for_int! { + u8: fwnode_property_read_u8_array, + u16: fwnode_property_read_u16_array, + u32: fwnode_property_read_u32_array, + u64: fwnode_property_read_u64_array, + i8: fwnode_property_read_u8_array, + i16: fwnode_property_read_u16_array, + i32: fwnode_property_read_u32_array, + i64: fwnode_property_read_u64_array, +} + +/// A helper for reading device properties. +/// +/// Use [`Self::required_by`] if a missing property is considered a bug and +/// [`Self::optional`] otherwise. +/// +/// For convenience, [`Self::or`] and [`Self::or_default`] are provided. +pub struct PropertyGuard<'fwnode, 'name, T> { + /// The result of reading the property. + inner: Result, + /// The fwnode of the property, used for logging in the "required" case. + fwnode: &'fwnode FwNode, + /// The name of the property, used for logging in the "required" case. + name: &'name CStr, +} + +impl PropertyGuard<'_, '_, T> { + /// Access the property, indicating it is required. + /// + /// If the property is not present, the error is automatically logged. If a + /// missing property is not an error, use [`Self::optional`] instead. The + /// device is required to associate the log with it. + pub fn required_by(self, dev: &super::Device) -> Result { + if self.inner.is_err() { + dev_err!( + dev, + "{}: property '{}' is missing\n", + self.fwnode, + self.name + ); + } + self.inner + } + + /// Access the property, indicating it is optional. + /// + /// In contrast to [`Self::required_by`], no error message is logged if + /// the property is not present. + pub fn optional(self) -> Option { + self.inner.ok() + } + + /// Access the property or the specified default value. + /// + /// Do not pass a sentinel value as default to detect a missing property. + /// Use [`Self::required_by`] or [`Self::optional`] instead. + pub fn or(self, default: T) -> T { + self.inner.unwrap_or(default) + } +} + +impl PropertyGuard<'_, '_, T> { + /// Access the property or a default value. + /// + /// Use [`Self::or`] to specify a custom default value. + pub fn or_default(self) -> T { + self.inner.unwrap_or_default() + } +} diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs new file mode 100644 index 000000000..c81fca5b4 --- /dev/null +++ b/rust/kernel/device_id.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic implementation of device IDs. +//! +//! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is +//! expected to implement [`RawDeviceId`]. + +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + +/// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that `Self` is layout-compatible with [`RawDeviceId::RawType`]; +/// i.e. it's safe to transmute to `RawType`. +/// +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. +/// +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; +} + +/// Extension trait for [`RawDeviceId`] for devices that embed an index or context value. +/// +/// This is typically used when the device ID struct includes a field like `driver_data` +/// that is used to store a pointer-sized value (e.g., an index or context pointer). +/// +/// # Safety +/// +/// Implementers must ensure that `DRIVER_DATA_OFFSET` is the correct offset (in bytes) to +/// the context/data field (e.g., the `driver_data` field) within the raw device ID structure. +/// This field must be correctly sized to hold a `usize`. +/// +/// Ideally, the data should be added during `Self` to `RawType` conversion, +/// but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceIdIndex: RawDeviceId { + /// The offset (in bytes) to the context/data field in the raw device ID. + const DRIVER_DATA_OFFSET: usize; + + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`. + unsafe fn info_unchecked(&self) -> &'static U { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, this is stored in `IdArray` so is + // convertible to `&'static U`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<&U>() + .read() + } + } + + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`, or has NULL (or 0) as driver data. + unsafe fn info_unchecked_opt(&self) -> Option<&'static U> { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, if this is stored in `IdArray`, this is + // convertible to `Option<&'static U>`. Otherwise it is NULL which is `None` as + // `Option<&U>`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::>() + .read() + } + } +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + // This is `MaybeUninit` so any bytes inside it can carry provenance in CTFE. + // If this were `T::RawType`, integer fields would not be able to contain pointers. + ids: [MaybeUninit; N], + sentinel: MaybeUninit, + phantom: PhantomData<&'static U>, +} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be sent between threads safely +// if `&U` can. +unsafe impl Send for IdArray {} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be shared between threads safely +// if `&U` can. +unsafe impl Sync for IdArray {} + +impl IdArray { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, &'static U); N]) -> Self { + let mut raw_ids = [const { MaybeUninit::::uninit() }; N]; + + let mut i = 0usize; + while i < N { + // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is + // layout-wise compatible with `RawType`. + raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, this would be effectively + // `raw_ids[i].driver_data = ids[i].1;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(T::DRIVER_DATA_OFFSET) + .cast::<&U>() + .write(ids[i].1); + } + + i += 1; + } + + core::mem::forget(ids); + + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } + } +} + +impl IdArray { + /// Creates a new instance of the array without writing index values. + /// + /// The contents are derived from the given identifiers and context information. + /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. + pub const fn new_without_index(ids: [T; N]) -> Self { + // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. + let raw_ids: [MaybeUninit; N] = unsafe { core::mem::transmute_copy(&ids) }; + core::mem::forget(ids); + + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } + } +} + +/// A device id table. +/// +/// This trait is only implemented by `IdArray`. +/// +/// The purpose of this trait is to allow `&'static dyn IdArray` to be in context when `N` in +/// `IdArray` doesn't matter. +pub trait IdTable { + /// Obtain the pointer to the ID table. + fn as_ptr(&self) -> *const T::RawType; +} + +impl IdTable for IdArray { + fn as_ptr(&self) -> *const T::RawType { + // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance + // to access the sentinel. + core::ptr::from_ref(self).cast() + } +} + +/// Create device table alias for modpost. +#[macro_export] +macro_rules! module_device_table { + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, $id_info_type: ty, + [$(($id: expr, $info:expr $(,)?)),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + $id_info_type, + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new([$(($id, &$info),)*]); + }; + + // Case for no ID info. + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, @none, + [$($id: expr),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + (), + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new_without_index([$($id),*]); + }; +} diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs new file mode 100644 index 000000000..d2924aaae --- /dev/null +++ b/rust/kernel/devres.rs @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devres abstraction +//! +//! [`Devres`] represents an abstraction for the kernel devres (device resource management) +//! implementation. + +use crate::{ + alloc::Flags, + bindings, + device::{ + Bound, + Device, // + }, + error::to_result, + prelude::*, + revocable::{ + Revocable, + RevocableGuard, // + }, + sync::{ + aref::ARef, + rcu, + Arc, + Completion, // + }, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // + }, +}; + +/// Inner type that embeds a `struct devres_node` and the `Revocable`. +#[repr(C)] +#[pin_data] +struct Inner { + #[pin] + node: Opaque, + #[pin] + data: Revocable, + #[pin] + revocation: Completion, +} + +/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to +/// manage their lifetime. +/// +/// [`Device`] bound resources should be freed when either the resource goes out of scope or the +/// [`Device`] is unbound respectively, depending on what happens first. In any case, it is always +/// guaranteed that revoking the device resource is completed before the corresponding [`Device`] +/// is unbound. +/// +/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the +/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). +/// +/// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource +/// anymore. +/// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// +/// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s +/// [`Drop`] implementation. +/// +/// # Examples +/// +/// ```no_run +/// # #![cfg(CONFIG_HAS_IOMEM)] +/// use kernel::{ +/// bindings, +/// device::{ +/// Bound, +/// Device, +/// }, +/// devres::Devres, +/// io::{ +/// Io, +/// IoBase, +/// Mmio, +/// MmioRaw, +/// MmioBackend, +/// PhysAddr, +/// Region, // +/// }, +/// prelude::*, +/// }; +/// use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(MmioRaw>); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. +/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; +/// } +/// } +/// +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem { +/// type Backend = MmioBackend; +/// type Target = Region; +/// +/// fn as_view(self) -> Mmio<'a, Region> { +/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. +/// unsafe { Mmio::from_raw(self.0) } +/// } +/// } +/// # fn no_run(dev: &Device) -> Result<(), Error> { +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// let devres = Devres::new(dev, iomem)?; +/// +/// let res = devres.try_access().ok_or(ENXIO)?; +/// res.write8(0x42, 0x0); +/// # Ok(()) +/// # } +/// ``` +pub struct Devres { + dev: ARef, + inner: Arc>, +} + +// Calling the FFI functions from the `base` module directly from the `Devres` impl may result in +// them being called directly from driver modules. This happens since the Rust compiler will use +// monomorphisation, so it might happen that functions are instantiated within the calling driver +// module. For now, work around this with `#[inline(never)]` helpers. +// +// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to +// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported. +mod base { + use kernel::{ + bindings, + prelude::*, // + }; + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_init( + node: *mut bindings::devres_node, + release: bindings::dr_node_release_t, + free: bindings::dr_node_free_t, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_init`. + unsafe { bindings::devres_node_init(node, release, free) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_set_node_dbginfo( + node: *mut bindings::devres_node, + name: *const c_char, + size: usize, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_set_node_dbginfo`. + unsafe { bindings::devres_set_node_dbginfo(node, name, size) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_add( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_add`. + unsafe { bindings::devres_node_add(dev, node) } + } + + #[must_use] + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_remove( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) -> bool { + // SAFETY: Safety requirements are the same as `bindings::devres_node_remove`. + unsafe { bindings::devres_node_remove(dev, node) } + } +} + +impl Devres { + /// Creates a new [`Devres`] instance of the given `data`. + /// + /// The `data` encapsulated within the returned `Devres` instance' `data` will be + /// (revoked)[`Revocable`] once the device is detached. + pub fn new(dev: &Device, data: impl PinInit) -> Result + where + Error: From, + { + let inner = Arc::pin_init::( + try_pin_init!(Inner { + node <- Opaque::ffi_init(|node: *mut bindings::devres_node| { + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_node_init( + node, + Some(Self::devres_node_release), + Some(Self::devres_node_free_node), + ) + }; + + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_set_node_dbginfo( + node, + // TODO: Use `core::any::type_name::()` once it is a `const fn`, + // such that we can convert the `&str` to a `&CStr` at compile-time. + c"Devres".as_char_ptr(), + core::mem::size_of::>(), + ) + }; + }), + data <- Revocable::new(data), + revocation <- Completion::new(), + }), + GFP_KERNEL, + )?; + + // SAFETY: + // - `dev` is a valid pointer to a bound `struct device`. + // - `node` is a valid pointer to a `struct devres_node`. + // - `devres_node_add()` is guaranteed not to call `devres_node_release()` for the entire + // lifetime of `dev`. + unsafe { base::devres_node_add(dev.as_raw(), inner.node.get()) }; + + // Take additional reference count for `devres_node_add()`. + core::mem::forget(inner.clone()); + + Ok(Self { + dev: dev.into(), + inner, + }) + } + + fn data(&self) -> &Revocable { + &self.inner.data + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_node_release( + _dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner, node) }; + + // SAFETY: `inner` is a valid `Inner` pointer. + let inner = unsafe { &*inner }; + + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_node_free_node(node: *mut bindings::devres_node) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner, node) }; + + // SAFETY: `inner` points to the entire `Inner` allocation. + drop(unsafe { Arc::from_raw(inner) }); + } + + fn remove_node(&self) -> bool { + // SAFETY: + // - `self.device().as_raw()` is a valid pointer to a bound `struct device`. + // - `self.inner.node.get()` is a valid pointer to a `struct devres_node`. + unsafe { base::devres_node_remove(self.device().as_raw(), self.inner.node.get()) } + } + + /// Return a reference of the [`Device`] this [`Devres`] instance has been created with. + pub fn device(&self) -> &Device { + &self.dev + } + + /// Obtain `&'a T`, bypassing the [`Revocable`]. + /// + /// This method allows to directly obtain a `&'a T`, bypassing the [`Revocable`], by presenting + /// a `&'a Device` of the same [`Device`] this [`Devres`] instance has been created with. + /// + /// # Errors + /// + /// An error is returned if `dev` does not match the same [`Device`] this [`Devres`] instance + /// has been created with. + /// + /// # Examples + /// + /// ```no_run + /// #![cfg(CONFIG_PCI)] + /// use kernel::{ + /// device::Core, + /// devres::Devres, + /// io::Io, + /// pci, // + /// }; + /// + /// fn from_core(dev: &pci::Device>, devres: Devres>) -> Result { + /// let bar = devres.access(dev.as_ref())?; + /// + /// let _ = bar.read32(0x0); + /// + /// // might_sleep() + /// + /// bar.write32(0x42, 0x0); + /// + /// Ok(()) + /// } + /// ``` + pub fn access<'a>(&'a self, dev: &'a Device) -> Result<&'a T> { + if self.dev.as_raw() != dev.as_raw() { + return Err(EINVAL); + } + + // SAFETY: `dev` being the same device as the device this `Devres` has been created for + // proves that `self.data` hasn't been revoked and is guaranteed to not be revoked as long + // as `dev` lives; `dev` lives at least as long as `self`. + Ok(unsafe { self.data().access() }) + } + + /// [`Devres`] accessor for [`Revocable::try_access`]. + pub fn try_access(&self) -> Option> { + self.data().try_access() + } + + /// [`Devres`] accessor for [`Revocable::try_access_with`]. + pub fn try_access_with R>(&self, f: F) -> Option { + self.data().try_access_with(f) + } + + /// [`Devres`] accessor for [`Revocable::try_access_with_guard`]. + pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> { + self.data().try_access_with_guard(guard) + } +} + +// SAFETY: `Devres` can be send to any task, if `T: Send`. +unsafe impl Send for Devres {} + +// SAFETY: `Devres` can be shared with any task, if `T: Sync`. +unsafe impl Sync for Devres {} + +impl Drop for Devres { + fn drop(&mut self) { + // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data + // anymore, hence it is safe not to wait for the grace period to finish. + if unsafe { self.data().revoke_nosync() } { + self.inner.revocation.complete_all(); + + // We revoked `self.data` before devres did, hence try to remove it. + if self.remove_node() { + // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` + // for `devres_node_add()`. Since `remove_node()` was successful, we have to drop + // this additional reference count. + drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); + } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); + } + } +} + +/// Guard returned by [`DevresLt::try_access`]. +/// +/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow +/// lifetime. +pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>); + +impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> { + type Target = F::Of<'a>; + + #[inline] + fn deref(&self) -> &Self::Target { + F::cast_ref(&*self.0) + } +} + +/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access. +/// +/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow +/// lifetime in all access methods. +/// +/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`], +/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors +/// ([`Self::access_with`], [`Self::try_access_with`]). +pub struct DevresLt(Devres>) +where + for<'a> F::Of<'a>: Send; + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Creates a new [`DevresLt`] instance of the given `data`. + /// + /// # Safety + /// + /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows + /// access until the device is unbound, which may outlast `'a`. + pub unsafe fn new<'a, E>( + dev: &'a Device, + data: impl PinInit, E>, + ) -> Result + where + Error: From, + { + // SAFETY: The caller guarantees the data is valid for the device's full bound scope. + // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical + // representation; casting the slot pointer is sound. + let data = unsafe { pin_init::cast_pin_init(data) }; + + Ok(Self(Devres::new(dev, data)?)) + } + + /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with. + #[inline] + pub fn device(&self) -> &Device { + self.0.device() + } + + /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure. + /// + /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any + /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`]. + #[inline] + pub fn access_with(&self, dev: &Device, f: G) -> Result + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.access(dev).map(f) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access_with`]. + #[inline] + pub fn try_access_with(&self, f: G) -> Option + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.data().try_access_with(f) + } +} + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`]. + /// + /// This method works like [`Devres::access`], but shortens the returned reference's lifetime + /// from `'static` to `'a` via [`CovariantForLt::cast_ref`]. + #[inline] + pub fn access<'a>(&'a self, dev: &'a Device) -> Result<&'a F::Of<'a>> { + self.0.access(dev).map(F::cast_ref) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access`]. + #[inline] + pub fn try_access(&self) -> Option> { + self.0.data().try_access().map(DevresGuard) + } +} + +/// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. +fn register_foreign

(dev: &Device, data: P) -> Result +where + P: ForeignOwnable + Send + 'static, +{ + let ptr = data.into_foreign(); + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn callback(ptr: *mut kernel::ffi::c_void) { + // SAFETY: `ptr` is the pointer to the `ForeignOwnable` leaked above and hence valid. + drop(unsafe { P::from_foreign(ptr.cast()) }); + } + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid and bound device. + // - `ptr` is a valid pointer the `ForeignOwnable` devres takes ownership of. + to_result(unsafe { + // `devm_add_action_or_reset()` also calls `callback` on failure, such that the + // `ForeignOwnable` is released eventually. + bindings::devm_add_action_or_reset(dev.as_raw(), Some(callback::

), ptr.cast()) + }) +} + +/// Encapsulate `data` in a [`KBox`] and [`Drop::drop`] `data` once `dev` is unbound. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::{ +/// device::{ +/// Bound, +/// Device, // +/// }, +/// devres, // +/// }; +/// +/// /// Registration of e.g. a class device, IRQ, etc. +/// struct Registration; +/// +/// impl Registration { +/// fn new() -> Self { +/// // register +/// +/// Self +/// } +/// } +/// +/// impl Drop for Registration { +/// fn drop(&mut self) { +/// // unregister +/// } +/// } +/// +/// fn from_bound_context(dev: &Device) -> Result { +/// devres::register(dev, Registration::new(), GFP_KERNEL) +/// } +/// ``` +pub fn register(dev: &Device, data: impl PinInit, flags: Flags) -> Result +where + T: Send + 'static, + Error: From, +{ + let data = KBox::pin_init(data, flags)?; + + register_foreign(dev, data) +} diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs new file mode 100644 index 000000000..2ce09f8e9 --- /dev/null +++ b/rust/kernel/dma.rs @@ -0,0 +1,1251 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Direct memory access (DMA). +//! +//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) + +use crate::{ + bindings, + debugfs, + device::{ + self, + Bound, + Core, // + }, + error::to_result, + fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, + prelude::*, + ptr::KnownSize, + sync::aref::ARef, + transmute::{ + AsBytes, + FromBytes, // + }, + uaccess::UserSliceWriter, // +}; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, // +}; + +/// DMA address type. +/// +/// Represents a bus address used for Direct Memory Access (DMA) operations. +/// +/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on +/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`. +/// +/// Note that this may be `u64` even on 32-bit architectures. +pub type DmaAddress = bindings::dma_addr_t; + +/// Trait to be implemented by DMA capable bus devices. +/// +/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, +/// where the underlying bus is DMA capable, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`](::kernel::platform::Device) +pub trait Device<'a>: AsRef>> { + /// Set up the device's DMA streaming addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA coherent addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA addressing capabilities. + /// + /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { + bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) + }) + } + + /// Set the maximum size of a single DMA segment the device may request. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`Coherent::zeroed`]. + unsafe fn dma_set_max_seg_size(&self, size: u32) { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this parameter. + unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) } + } +} + +/// A DMA mask that holds a bitmask with the lowest `n` bits set. +/// +/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values +/// are guaranteed to never exceed the bit width of `u64`. +/// +/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaMask(u64); + +impl DmaMask { + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, results in a build error. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::new::<0>(); + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::new::<1>(); + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::new::<64>(); + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// // Build failure. + /// // let mask_overflow = DmaMask::new::<100>(); + /// ``` + #[inline] + pub const fn new() -> Self { + let Ok(mask) = Self::try_new(N) else { + build_error!("Invalid DMA Mask."); + }; + + mask + } + + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, returns [`EINVAL`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::try_new(0)?; + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::try_new(1)?; + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::try_new(64)?; + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// let mask_overflow = DmaMask::try_new(100); + /// assert!(mask_overflow.is_err()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub const fn try_new(n: u32) -> Result { + Ok(Self(match n { + 0 => 0, + 1..=64 => u64::MAX >> (64 - n), + _ => return Err(EINVAL), + })) + } + + /// Returns the underlying `u64` bitmask value. + #[inline] + pub const fn value(&self) -> u64 { + self.0 + } +} + +/// Possible attributes associated with a DMA mapping. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`attrs`] module. +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::{Bound, Device}; +/// use kernel::dma::{attrs::*, Coherent}; +/// +/// # fn test(dev: &Device) -> Result { +/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; +/// let c: Coherent<[u64]> = +/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// # Ok::<(), Error>(()) } +/// ``` +#[derive(Clone, Copy, PartialEq)] +#[repr(transparent)] +pub struct Attrs(u32); + +impl Attrs { + /// Get the raw representation of this attribute. + pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { + self.0 as crate::ffi::c_ulong + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Attrs) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Attrs { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Attrs { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Attrs { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// DMA mapping attributes. +pub mod attrs { + use super::Attrs; + + /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads + /// and writes may pass each other. + pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); + + /// Specifies that writes to the mapping may be buffered to improve performance. + pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); + + /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming + /// that it has been already transferred to 'device' domain. + pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); + + /// Forces contiguous allocation of the buffer in physical memory. + pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); + + /// Hints DMA-mapping subsystem that it's probably not worth the time to try + /// to allocate memory to in a way that gives better TLB efficiency. + pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); + + /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to + /// `__GFP_NOWARN`). + pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); + + /// Indicates that the buffer is fully accessible at an elevated privilege level (and + /// ideally inaccessible or at least read-only at lesser-privileged levels). + pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); + + /// Indicates that the buffer is MMIO memory. + pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO); +} + +/// DMA data direction. +/// +/// Corresponds to the C [`enum dma_data_direction`]. +/// +/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum DataDirection { + /// The DMA mapping is for bidirectional data transfer. + /// + /// This is used when the buffer can be both read from and written to by the device. + /// The cache for the corresponding memory region is both flushed and invalidated. + Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL), + + /// The DMA mapping is for data transfer from memory to the device (write). + /// + /// The CPU has prepared data in the buffer, and the device will read it. + /// The cache for the corresponding memory region is flushed before device access. + ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE), + + /// The DMA mapping is for data transfer from the device to memory (read). + /// + /// The device will write data into the buffer for the CPU to read. + /// The cache for the corresponding memory region is invalidated before CPU access. + FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE), + + /// The DMA mapping is not for data transfer. + /// + /// This is primarily for debugging purposes. With this direction, the DMA mapping API + /// will not perform any cache coherency operations. + None = Self::const_cast(bindings::dma_data_direction_DMA_NONE), +} + +impl DataDirection { + /// Casts the bindgen-generated enum type to a `u32` at compile time. + /// + /// This function will cause a compile-time error if the underlying value of the + /// C enum is out of bounds for `u32`. + const fn const_cast(val: bindings::dma_data_direction) -> u32 { + // CAST: The C standard allows compilers to choose different integer types for enums. + // To safely check the value, we cast it to a wide signed integer type (`i128`) + // which can hold any standard C integer enum type without truncation. + let wide_val = val as i128; + + // Check if the value is outside the valid range for the target type `u32`. + // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison. + if wide_val < 0 || wide_val > u32::MAX as i128 { + // Trigger a compile-time error in a const context. + build_error!("C enum value is out of bounds for the target type `u32`."); + } + + // CAST: This cast is valid because the check above guarantees that `wide_val` + // is within the representable range of `u32`. + wide_val as u32 + } +} + +impl From for bindings::dma_data_direction { + /// Returns the raw representation of [`enum dma_data_direction`]. + fn from(direction: DataDirection) -> Self { + // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum. + // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible + // with the enum variants of `DataDirection`, which is a valid assumption given our + // compile-time checks. + direction as u32 as Self + } +} + +/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object. +/// +/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA +/// address is not exposed and it cannot be accessed by a device. This means it can safely be used +/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe). +/// +/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then +/// convert it into a [`Coherent`] object to share it with the device. +/// +/// # Examples +/// +/// `CoherentBox`: +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// *dmem = 42; +/// let dmem: Coherent = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +/// +/// `CoherentBox<[T]>`: +/// +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// dmem.fill(42); +/// let dmem: Coherent<[u64]> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +pub struct CoherentBox(Coherent); + +impl CoherentBox<[T]> { + /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Initializes the element at `i` using the given initializer. + /// + /// Returns `EINVAL` if `i` is out of bounds. + pub fn init_at(&mut self, i: usize, init: impl Init) -> Result + where + Error: From, + { + if i >= self.0.len() { + return Err(EINVAL); + } + + let ptr = &raw mut self[i]; + + // SAFETY: + // - `ptr` is valid, properly aligned, and within this allocation. + // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on + // error cannot leave the element in an invalid state. + // - The DMA address has not been exposed yet, so there is no concurrent device access. + unsafe { pin_init::raw_try_init(ptr, init)? }; + + Ok(()) + } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`]. + /// + /// # Examples + /// + /// ``` + /// use core::ops::Deref; + /// + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// CoherentBox + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// let c: CoherentBox<[u8]> = + /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// assert_eq!(c.deref(), &data); + /// # Ok::<(), Error>(()) } + /// ``` + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result + where + T: Copy, + { + let mut slice = Self(Coherent::::alloc_slice_with_attrs( + dev, + data.len(), + gfp_flags, + dma_attrs, + )?); + + // PANIC: `slice` was created with length `data.len()`. + slice.copy_from_slice(data); + + Ok(slice) + } + + /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } +} + +impl CoherentBox { + /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [`CoherentBox::zeroed_slice`], but for a single element. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } +} + +impl Deref for CoherentBox { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for CoherentBox { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_mut() } + } +} + +impl From> for Coherent { + #[inline] + fn from(value: CoherentBox) -> Self { + value.0 + } +} + +/// An abstraction of the `dma_alloc_coherent` API. +/// +/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map +/// large coherent DMA regions. +/// +/// A [`Coherent`] instance contains a pointer to the allocated region (in the +/// processor's virtual address space) and the device address which can be given to the device +/// as the DMA address base of the region. The region is released once [`Coherent`] +/// is dropped. +/// +/// # Invariants +/// +/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer +/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the +/// region. +/// - The size in bytes of the allocation is equal to size information via pointer. +// TODO +// +// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness +// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure +// that device resources can never survive device unbind. +// +// However, it is neither desirable nor necessary to protect the allocated memory of the DMA +// allocation from surviving device unbind; it would require RCU read side critical sections to +// access the memory, which may require subsequent unnecessary copies. +// +// Hence, find a way to revoke the device resources of a `Coherent`, but not the +// entire `Coherent` including the allocated memory itself. +pub struct Coherent { + dev: ARef, + dma_addr: DmaAddress, + cpu_addr: NonNull, + dma_attrs: Attrs, +} + +impl Coherent { + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + T::size(self.cpu_addr.as_ptr()) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.cpu_addr.as_ptr() + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + self.cpu_addr.as_ptr() + } + + /// Returns a DMA address which may be given to the device as the base of the region. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + #[inline] + pub unsafe fn as_ref(&self) -> &T { + // SAFETY: per safety requirement. + unsafe { &*self.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + #[expect(clippy::mut_from_ref, reason = "unsafe to use API")] + #[inline] + pub unsafe fn as_mut(&self) -> &mut T { + // SAFETY: per safety requirement. + unsafe { &mut *self.as_mut_ptr() } + } +} + +impl Coherent { + /// Allocates a region of `T` of coherent memory. + fn alloc_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + let mut dma_addr = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + core::mem::size_of::(), + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?; + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for `T`, + // hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_addr, + cpu_addr, + dma_attrs, + }) + } + + /// Allocates a region of type `T` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64; 4]> = + /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } + + /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is + /// initialized with `init`. + pub fn init_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + init: impl Init, + ) -> Result + where + Error: From, + { + let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?; + let ptr = dmem.as_mut_ptr(); + + // SAFETY: + // - `ptr` is valid, properly aligned, and points to exclusively owned memory. + // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying + // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` + // requirements we are bypassing. + unsafe { pin_init::raw_try_init(ptr, init)? }; + + Ok(dmem) + } + + /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized + /// with `init`. + #[inline] + pub fn init( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + init: impl Init, + ) -> Result + where + Error: From, + { + Self::init_with_attrs(dev, gfp_flags, Attrs(0), init) + } + + /// Allocates a region of `[T; len]` of coherent memory. + fn alloc_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + // `dma_alloc_attrs` cannot handle zero-length allocation, bail early. + if len == 0 { + Err(EINVAL)?; + } + + let size = core::mem::size_of::().checked_mul(len).ok_or(ENOMEM)?; + let mut dma_addr = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len); + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for + // `[T; len]`, hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Coherent { + dev: dev.into(), + dma_addr, + cpu_addr, + dma_attrs, + }) + } + + /// Allocates a zeroed region of type `T` of coherent memory. + /// + /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::::zeroed_slices` support + /// a runtime length. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64]> = + /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) + } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// // `c` has the same content as `data`. + /// let c: Coherent<[u8]> = + /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> + where + T: Copy, + { + CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into) + } + + /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } +} + +impl Coherent<[T]> { + /// Returns the number of elements `T` in this allocation. + /// + /// Note that this is not the size of the allocation in bytes, which is provided by + /// [`Self::size`]. + #[inline] + #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")] + pub fn len(&self) -> usize { + self.cpu_addr.len() + } +} + +/// Note that the device configured to do DMA must be halted before this object is dropped. +impl Drop for Coherent { + fn drop(&mut self) { + let size = T::size(self.cpu_addr.as_ptr()); + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // The cpu address, and the dma address are valid due to the type invariants on + // `Coherent`. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + size, + self.cpu_addr.as_ptr().cast(), + self.dma_addr, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: It is safe to send a `Coherent` to another thread if `T` +// can be sent to another thread. +unsafe impl Send for Coherent {} + +// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all +// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, +// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. +// The safe methods only return metadata or raw pointers whose use requires `unsafe`. +unsafe impl Sync for Coherent {} + +impl debugfs::BinaryWriter for Coherent { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + // If the offset is too large for a usize (e.g. on 32-bit platforms), + // then consider that as past EOF and just return 0 bytes. + let Ok(offset_val) = usize::try_from(*offset) else { + return Ok(0); + }; + + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); + + writer.write_dma(self, offset_val, count)?; + + *offset += count as i64; + Ok(count) + } +} + +/// An opaque DMA allocation without a kernel virtual mapping. +/// +/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory. +/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel +/// virtual mapping is created for the buffer. The value returned by the C API as the CPU +/// address is an opaque handle used only to free the allocation. +/// +/// This is useful for buffers that are only ever accessed by hardware. +/// +/// # Invariants +/// +/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with +/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. +/// - `dma_addr` is the corresponding bus address for device DMA. +/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. +/// - `dma_attrs` contains the attributes used for the allocation, always including +/// `DMA_ATTR_NO_KERNEL_MAPPING`. +pub struct CoherentHandle { + dev: ARef, + dma_addr: DmaAddress, + cpu_handle: NonNull, + size: usize, + dma_attrs: Attrs, +} + +impl CoherentHandle { + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + /// + /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is + /// always set implicitly. + /// + /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. + pub fn alloc_with_attrs( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + if size == 0 { + return Err(EINVAL); + } + + let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + let mut dma_addr = 0; + // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. + let cpu_handle = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + + let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; + + // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, + // and we hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_addr, + cpu_handle, + size, + dma_attrs, + }) + } + + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + #[inline] + pub fn alloc( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) + } + + /// Returns the DMA address for this allocation. + /// + /// This address can be programmed into device hardware for DMA access. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for CoherentHandle { + fn drop(&mut self) { + // SAFETY: All values are valid by the type invariants on `CoherentHandle`. + // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + self.size, + self.cpu_handle.as_ptr(), + self.dma_addr, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, +// and a size. None of these are tied to a specific thread. +unsafe impl Send for CoherentHandle {} + +// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only +// operations on `&CoherentHandle` are reading the DMA address and size, both of which are +// plain `Copy` values. +unsafe impl Sync for CoherentHandle {} + +/// View type for `Coherent`. +/// +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA +/// address. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_addr: DmaAddress, +} + +impl Copy for CoherentView<'_, T> {} +impl Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns the DMA address which may be given to the device as base of the region. + #[inline] + pub fn dma_address(self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_addr = view.dma_addr + offset as DmaAddress; + CoherentView { + dma_addr, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl IoCapable for CoherentIoBackend +where + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_addr: self.dma_addr, + } + } +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 000000000..c9c74c4dd --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! This documentation describes how to implement a bus specific driver API and how to align it with +//! the design of (bus specific) devices. +//! +//! Note: Readers are expected to know the content of the documentation of [`Device`] and +//! [`DeviceContext`]. +//! +//! # Driver Trait +//! +//! The main driver interface is defined by a bus specific driver trait. For instance: +//! +//! ```ignore +//! pub trait Driver { +//! /// The type holding information about each device ID supported by the driver. +//! type IdInfo: 'static; +//! +//! /// The type of the driver's bus device private data. +//! type Data<'bound>: Send + 'bound; +//! +//! /// The table of OF device ids supported by the driver. +//! const OF_ID_TABLE: Option> = None; +//! +//! /// The table of ACPI device ids supported by the driver. +//! const ACPI_ID_TABLE: Option> = None; +//! +//! /// Driver probe. +//! fn probe<'bound>( +//! dev: &'bound Device>, +//! id_info: &'bound Self::IdInfo, +//! ) -> impl PinInit, Error> + 'bound; +//! +//! /// Driver unbind (optional). +//! fn unbind<'bound>( +//! dev: &'bound Device>, +//! this: Pin<&Self::Data<'bound>>, +//! ) { +//! let _ = (dev, this); +//! } +//! } +//! ``` +//! +//! For specific examples see: +//! +//! * [`platform::Driver`](kernel::platform::Driver) +#![cfg_attr( + CONFIG_AUXILIARY_BUS, + doc = "* [`auxiliary::Driver`](kernel::auxiliary::Driver)" +)] +#![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] +//! +//! The `probe()` callback should return a +//! `impl PinInit, Error>`, i.e. the driver's private data. The bus +//! abstraction should store the pointer in the corresponding bus device. The generic +//! [`Device`] infrastructure provides common helpers for this purpose on its +//! [`Device`] implementation. +//! +//! All driver callbacks should provide a reference to the driver's private data. Once the driver +//! is unbound from the device, the bus abstraction should take back the ownership of the driver's +//! private data from the corresponding [`Device`] and [`drop`] it. +//! +//! All driver callbacks should provide a [`Device`] reference (see also [`device::Core`]). +//! +//! # Adapter +//! +//! The adapter implementation of a bus represents the abstraction layer between the C bus +//! callbacks and the Rust bus callbacks. It therefore has to be generic over an implementation of +//! the [driver trait](#driver-trait). +//! +//! ```ignore +//! pub struct Adapter; +//! ``` +//! +//! There's a common [`Adapter`] trait that can be implemented to inherit common driver +//! infrastructure, such as finding the ID info from an [`of::IdTable`] or [`acpi::IdTable`]. +//! +//! # Driver Registration +//! +//! In order to register C driver types (such as `struct platform_driver`) the [adapter](#adapter) +//! should implement the [`RegistrationOps`] trait. +//! +//! This trait implementation can be used to create the actual registration with the common +//! [`Registration`] type. +//! +//! Typically, bus abstractions want to provide a bus specific `module_bus_driver!` macro, which +//! creates a kernel module with exactly one [`Registration`] for the bus specific adapter. +//! +//! The generic driver infrastructure provides a helper for this with the [`module_driver`] macro. +//! +//! # Device IDs +//! +//! Besides the common device ID types, such as [`of::DeviceId`] and [`acpi::DeviceId`], most buses +//! may need to implement their own device ID types. +//! +//! For this purpose the generic infrastructure in [`device_id`] should be used. +//! +//! [`Core`]: device::Core +//! [`Device`]: device::Device +//! [`Device`]: device::Device +//! [`Device`]: device::Device +//! [`DeviceContext`]: device::DeviceContext +//! [`device_id`]: kernel::device_id +//! [`module_driver`]: kernel::module_driver + +use crate::{ + acpi, + device, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; + +/// Trait describing the layout of a specific device driver. +/// +/// This trait describes the layout of a specific driver structure, such as `struct pci_driver` or +/// `struct platform_driver`. +/// +/// # Safety +/// +/// Implementors must guarantee that: +/// - `DriverType` is `repr(C)`, +/// - `DriverData` is the type of the driver's device private data. +/// - `DriverType` embeds a valid `struct device_driver` at byte offset `DEVICE_DRIVER_OFFSET`. +pub unsafe trait DriverLayout { + /// The specific driver type embedding a `struct device_driver`. + type DriverType: Default; + + /// The type of the driver's bus device private data. + type DriverData<'bound>; + + /// Byte offset of the embedded `struct device_driver` within `DriverType`. + /// + /// This must correspond exactly to the location of the embedded `struct device_driver` field. + const DEVICE_DRIVER_OFFSET: usize; +} + +/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, +/// Amba, etc.) to provide the corresponding subsystem specific implementation to register / +/// unregister a driver of the particular type (`DriverType`). +/// +/// For instance, the PCI subsystem would set `DriverType` to `bindings::pci_driver` and call +/// `bindings::__pci_register_driver` from `RegistrationOps::register` and +/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. +/// +/// # Safety +/// +/// A call to [`RegistrationOps::unregister`] for a given instance of `DriverType` is only valid if +/// a preceding call to [`RegistrationOps::register`] has been successful. +pub unsafe trait RegistrationOps: DriverLayout { + /// Registers a driver. + /// + /// # Safety + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`RegistrationOps::unregister`]. + unsafe fn register( + reg: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`RegistrationOps::register`]. + /// + /// # Safety + /// + /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for + /// the same `reg`. + unsafe fn unregister(reg: &Opaque); +} + +/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. +/// `bindings::pci_driver`). Therefore a [`Registration`] must be initialized with a type that +/// implements the [`RegistrationOps`] trait, such that the generic `T::register` and +/// `T::unregister` calls result in the subsystem specific registration calls. +/// +///Once the `Registration` structure is dropped, the driver is unregistered. +#[pin_data(PinnedDrop)] +pub struct Registration { + #[pin] + reg: Opaque, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl Sync for Registration {} + +// SAFETY: Both registration and unregistration are implemented in C and safe to be performed from +// any thread, so `Registration` is `Send`. +unsafe impl Send for Registration {} + +impl Registration { + extern "C" fn post_unbind_callback(dev: *mut bindings::device) { + // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to + // a `struct device`. + // + // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. + let dev = unsafe { &*dev.cast::>>() }; + + // `remove()` has been completed at this point; devres resources are still valid and will + // be released after the driver's bus device private data is dropped. + // + // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the + // driver's bus device private data type. + drop(unsafe { dev.drvdata_obtain::>() }); + } + + /// Attach generic `struct device_driver` callbacks. + fn callbacks_attach(drv: &Opaque) { + let ptr = drv.get().cast::(); + + // SAFETY: + // - `drv.get()` yields a valid pointer to `Self::DriverType`. + // - Adding `DEVICE_DRIVER_OFFSET` yields the address of the embedded `struct device_driver` + // as guaranteed by the safety requirements of the `Driver` trait. + let base = unsafe { ptr.add(T::DEVICE_DRIVER_OFFSET) }; + + // CAST: `base` points to the offset of the embedded `struct device_driver`. + let base = base.cast::(); + + // SAFETY: It is safe to set the fields of `struct device_driver` on initialization. + unsafe { (*base).p_cb.post_unbind_rust = Some(Self::post_unbind_callback) }; + } + + /// Creates a new instance of the registration object. + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit + where + T: 'static, + { + try_pin_init!(Self { + reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| { + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. + unsafe { ptr.write(T::DriverType::default()) }; + + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has + // just been initialised above, so it's also valid for read. + let drv = unsafe { &*(ptr as *const Opaque) }; + + Self::callbacks_attach(drv); + + // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`. + unsafe { T::register(drv, name, module) } + }), + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The existence of `self` guarantees that `self.reg` has previously been + // successfully registered with `T::register` + unsafe { T::unregister(&self.reg) }; + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + + #[$crate::prelude::pin_data] + struct DriverModule { + #[pin] + _driver: $crate::driver::Registration>, + } + + impl $crate::InPlaceModule for DriverModule { + fn init( + module: &'static $crate::ThisModule + ) -> impl ::pin_init::PinInit { + $crate::try_pin_init!(Self { + _driver <- $crate::driver::Registration::new( + ::NAME, + module, + ), + }) + } + } + + $crate::prelude::module! { + type: DriverModule, + $($f)* + } + } +} + +/// The bus independent adapter to match a drivers and a devices. +/// +/// This trait should be implemented by the bus specific adapter, which represents the connection +/// of a device and a driver. +/// +/// It provides bus independent functions for device / driver interactions. +pub trait Adapter { + /// The type holding driver private data about each device id supported by the driver. + type IdInfo: 'static; + + /// The [`acpi::IdTable`] of the corresponding driver + fn acpi_id_table() -> Option>; + + /// The [`of::IdTable`] of the corresponding driver. + fn of_id_table() -> Option>; + + /// Returns the driver's private data from the matching entry of any of the ID tables, if any. + /// + /// If this returns `None`, it means that there is no match in any of the ID tables directly + /// associated with a [`device::Device`]. + /// + /// # Safety + /// + /// The caller must ensure that the `dev` matched data is of type `Self::IdInfo`. + #[inline] + unsafe fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + // SAFETY: `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let data = unsafe { bindings::device_get_match_data(dev.as_raw()) }; + + // SAFETY: Per safety requirement, `data` is of type `Self::IdInfo`. + unsafe { data.cast::().as_ref() } + } +} diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs new file mode 100644 index 000000000..6b88ade28 --- /dev/null +++ b/rust/kernel/drm/device.rs @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM device. +//! +//! C header: [`include/drm/drm_device.h`](srctree/include/drm/drm_device.h) + +use crate::{ + alloc::allocator::Kmalloc, + bindings, + device, + drm::{ + self, + driver::AllocImpl, + private::Sealed, // + }, + error::from_err_ptr, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, + workqueue::{ + HasDelayedWork, + HasWork, + Work, + WorkItem, // + }, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, +}; + +#[cfg(CONFIG_DRM_LEGACY)] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),*, + firstopen: None, + preclose: None, + dma_ioctl: None, + dma_quiescent: None, + context_dtor: None, + irq_handler: None, + irq_preinstall: None, + irq_postinstall: None, + irq_uninstall: None, + get_vblank_counter: None, + enable_vblank: None, + disable_vblank: None, + dev_priv_size: 0, + } + } +} + +#[cfg(not(CONFIG_DRM_LEGACY))] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),* + } + } +} + +/// A trait implemented by all possible contexts a [`Device`] can be used in. +/// +/// A [`Device`] can be in one of the following contexts: +/// +/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may +/// or may not be registered with userspace. +/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl +/// dispatch context. +/// - [`Registered`]: The device is currently registered with userspace and the parent bus device +/// is bound. +/// +/// Both `Device` and `Device` dereference to `Device` ([`Normal`]), +/// so any method available on a [`Normal`] device is also available in the other contexts. +pub trait DeviceContext: Sealed + Send + Sync + 'static {} + +/// The general-purpose, reference-counted [`DeviceContext`]. +/// +/// A [`Device`] in this context may or may not be registered with userspace. This context is used +/// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. +/// +/// [`AlwaysRefCounted`] is only implemented for `Device`, making this the required +/// context for [`ARef`]-based device handles. +pub struct Normal; + +impl Sealed for Normal {} +impl DeviceContext for Normal {} + +/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace. +/// +/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is +/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a +/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// The parent bus device is bound for the duration of any reference to a `Device`. +pub struct Registered; + +impl Sealed for Registered {} +impl DeviceContext for Registered {} + +/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously. +/// +/// A [`Device`] in this context has been registered at some point, but may be concurrently +/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the +/// device remains registered for the duration of the critical section. +/// +/// # Invariants +/// +/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at +/// some point. +pub struct Ioctl; + +impl Sealed for Ioctl {} +impl DeviceContext for Ioctl {} + +/// A [`Device`] which is known at compile-time to be unregistered with userspace. +/// +/// This type allows performing operations which are only safe to do before userspace registration, +/// and can be used to create a [`Registration`](drm::driver::Registration) once the driver is ready +/// to register the device with userspace. +/// +/// Since DRM device initialization must be single-threaded, this object is not thread-safe. +/// +/// # Invariants +/// +/// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been +/// registered with userspace until this type is dropped. +pub struct UnregisteredDevice(ARef>, NotThreadSafe); + +impl Deref for UnregisteredDevice { + type Target = Device; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl UnregisteredDevice { + const fn compute_features() -> u32 { + let mut features = drm::driver::FEAT_GEM; + + if T::FEAT_RENDER { + features |= drm::driver::FEAT_RENDER; + } + + features + } + + const VTABLE: bindings::drm_driver = drm_legacy_fields! { + load: None, + open: Some(drm::File::::open_callback), + postclose: Some(drm::File::::postclose_callback), + unload: None, + release: Some(Device::::release), + master_set: None, + master_drop: None, + debugfs_init: None, + + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, + + show_fdinfo: None, + fbdev_probe: None, + + major: T::INFO.major, + minor: T::INFO.minor, + patchlevel: T::INFO.patchlevel, + name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), + desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), + + driver_features: Self::compute_features(), + ioctls: T::IOCTLS.as_ptr(), + num_ioctls: T::IOCTLS.len() as i32, + fops: &Self::GEM_FOPS, + }; + + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::().as_ptr()); + + /// Create a new `UnregisteredDevice` for a `drm::Driver`. + /// + /// This can be used to create a [`Registration`](kernel::drm::Registration). + pub fn new( + dev: &T::ParentDevice, + data: impl PinInit, + ) -> Result { + // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` + // compatible `Layout`. + let layout = Kmalloc::aligned_layout(Layout::new::>()); + + // Use a temporary vtable without a `release` callback until `data` is initialized, so + // init failure can release the DRM device without dropping uninitialized fields. + let alloc_vtable = bindings::drm_driver { + release: None, + ..Self::VTABLE + }; + + // SAFETY: + // - `alloc_vtable` reference remains valid until no longer used, + // - `dev` is valid by its type invarants, + let raw_drm: *mut Device = unsafe { + bindings::__drm_dev_alloc( + dev.as_ref().as_raw(), + &alloc_vtable, + layout.size(), + mem::offset_of!(Device, dev), + ) + } + .cast(); + let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?; + + // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was + // successful. + let drm_dev = unsafe { Device::into_drm_device(raw_drm) }; + + // SAFETY: `raw_drm` is a valid pointer to `Self`. + let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) }; + + // SAFETY: + // - `raw_data` is a valid pointer to uninitialized memory. + // - `raw_data` will not move until it is dropped. + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { + // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the + // refcount must be non-zero. + unsafe { bindings::drm_dev_put(drm_dev) }; + })?; + + // SAFETY: `drm_dev` is still private to this function. + unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; + + // SAFETY: `raw_drm` is valid; no concurrent access before registration. + unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) }; + + // SAFETY: The reference count is one, and now we take ownership of that reference as a + // `drm::Device`. + // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`. + // `Self` cannot be copied or sent to another thread - ensuring that `drm_dev_register` + // won't be called during its lifetime and that the device is unregistered. + Ok(Self(unsafe { ARef::from_raw(raw_drm) }, NotThreadSafe)) + } +} + +/// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`]. +/// +/// A device in the [`Registered`] context is currently registered with userspace and its parent +/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context. +/// +/// # Invariants +/// +/// * `self.dev` is a valid instance of a `struct device`. +/// * The data layout of `Self` remains the same across all implementations of `C`. +/// * Any invariants for `C` also apply. +#[repr(C)] +pub struct Device { + dev: Opaque, + data: T::Data, + pub(super) registration_data: UnsafeCell>>, + _ctx: PhantomData, +} + +impl Device { + pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { + self.dev.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. + unsafe fn from_drm_device(ptr: *const bindings::drm_device) -> *mut Self { + // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a + // `struct drm_device` embedded in `Self`. + unsafe { crate::container_of!(Opaque::cast_from(ptr), Self, dev) }.cast_mut() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to `Self`. + unsafe fn into_drm_device(ptr: NonNull) -> *mut bindings::drm_device { + // SAFETY: By the safety requirements of this function, `ptr` is a valid pointer to `Self`. + unsafe { &raw mut (*ptr.as_ptr()).dev }.cast() + } + + /// Not intended to be called externally, except via declare_drm_ioctls!() + /// + /// # Safety + /// + /// * Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points + /// to can't drop to zero, for the duration of this function call and the entire duration when + /// the returned reference exists. + /// * Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is + /// embedded in `Self`. + /// * Callers promise that any type invariants of `C` will be upheld. + #[doc(hidden)] + pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { + // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a + // `struct drm_device` embedded in `Self`. + let ptr = unsafe { Self::from_drm_device(ptr) }; + + // SAFETY: `ptr` is valid by the safety requirements of this function. + unsafe { &*ptr.cast() } + } + + extern "C" fn release(ptr: *mut bindings::drm_device) { + // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. + let this = unsafe { Self::from_drm_device(ptr) }; + + // SAFETY: + // - When `release` runs it is guaranteed that there is no further access to `this`. + // - `this` is valid for dropping. + unsafe { core::ptr::drop_in_place(this) }; + } + + /// Change the [`DeviceContext`] for a [`Device`]. + /// + /// # Safety + /// + /// The caller promises that `self` fulfills all of the guarantees provided by the given + /// [`DeviceContext`]. + pub(crate) unsafe fn assume_ctx(&self) -> &Device { + // SAFETY: The data layout is identical via our type invariants. + unsafe { mem::transmute(self) } + } +} + +impl Device { + /// Guard against the parent bus device being unbound. + /// + /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise. + /// + /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound. + #[must_use] + pub fn registration_guard(&self) -> Option> { + let mut idx: i32 = 0; + // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`. + if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } { + // INVARIANT: + // - `idx` is the SRCU index from the successful `drm_dev_enter()` above. + // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning + // `drm_dev_unplug()` has not completed; since it is only called from + // `Registration::drop()` during parent unbind, the parent is still bound. + Some(RegistrationGuard { + // SAFETY: See INVARIANT above; the `Registered` context invariant holds. + dev: unsafe { self.assume_ctx() }, + idx, + _not_send: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard proving the DRM device is registered and the parent bus device is bound. +/// +/// The guard dereferences to [`Device`], providing access to the DRM device with +/// the guarantee that the parent bus device is bound for the entire duration of the critical +/// section. +/// +/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call. +/// - The parent bus device of `dev` is bound for the lifetime of this guard. +#[must_use] +pub struct RegistrationGuard<'a, T: drm::Driver> { + dev: &'a Device, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Device { + /// Returns a reference to the registration data with lifetime shortened from `'static`. + /// + /// # Safety + /// + /// The returned reference must not be exposed to code that can choose a concrete lifetime for + /// it, as that would be unsound for types that are invariant over their lifetime parameter + /// (e.g. it must be passed through an HRTB-bounded closure). + #[inline] + unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> { + // SAFETY: + // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid. + // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes + // are erased at runtime. + // - Caller guarantees the reference is only used behind an HRTB, making the lifetime + // shortening sound regardless of variance. + unsafe { (*self.registration_data.get()).cast::<_>().as_ref() } + } + + /// Access the registration data through a closure, with the lifetime tied to the closure + /// scope. + /// + /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid + /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop` + /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to + /// complete. + #[inline] + pub fn registration_data_with(&self, f: F) -> R + where + F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R, + { + // SAFETY: `Registered` guarantees the device is registered and the parent bus device is + // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references + // with a concrete short lifetime, satisfying the lifetime requirement of + // `registration_data_unchecked`. + f(unsafe { self.registration_data_unchecked() }) + } +} + +impl Deref for RegistrationGuard<'_, T> { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + self.dev + } +} + +impl Drop for RegistrationGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed + // by the type invariants of `RegistrationGuard`. + unsafe { bindings::drm_dev_exit(self.idx) }; + } +} + +impl Deref for Device { + type Target = T::Data; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +// SAFETY: DRM device objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::drm_dev_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: `obj` is a valid pointer to `Self`. + let drm_dev = unsafe { Self::into_drm_device(obj) }; + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_dev_put(drm_dev) }; + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &T::ParentDevice { + // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, + // which is guaranteed by the type invariant. + let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &T::ParentDevice { + let dev = (**self).as_ref().as_ref(); + + // SAFETY: A `Device` guarantees that the parent device is bound. + let dev = unsafe { dev.as_bound() }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +// SAFETY: A `drm::Device` can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected +// by the synchronization in `struct drm_device`. +unsafe impl Sync for Device {} + +impl WorkItem for Device +where + T::Data: WorkItem>, + T::Data: HasWork, +{ + type Pointer = ARef; + + fn run(ptr: ARef) { + T::Data::run(ptr); + } +} + +// SAFETY: +// +// - `raw_get_work` and `work_container_of` return valid pointers by relying on +// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is +// stored inline in `drm::Device`, so the `container_of` call is valid. +// +// - The two methods are true inverses of each other: given `ptr: *mut +// Device`, `raw_get_work` will return a `*mut Work, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, +// `work_container_of` will return a `*mut Device` through `container_of`. +unsafe impl HasWork for Device +where + T: drm::Driver, + T::Data: HasWork, + C: DeviceContext, +{ + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work { + // SAFETY: The caller promises that `ptr` points to a valid `Device`. + let data_ptr = unsafe { &raw mut (*ptr).data }; + + // SAFETY: `data_ptr` is a valid pointer to `T::Data`. + unsafe { T::Data::raw_get_work(data_ptr) } + } + + unsafe fn work_container_of(ptr: *mut Work) -> *mut Self { + // SAFETY: The caller promises that `ptr` points at a `Work` field in + // `T::Data`. + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + + // SAFETY: `T::Data` is stored as the `data` field in `Device`. + unsafe { crate::container_of!(data_ptr, Self, data) } + } +} + +// SAFETY: Our `HasWork` implementation returns a `work_struct` that is +// stored in the `work` field of a `delayed_work` with the same access rules as +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, +// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that +// is inside a `delayed_work`. +unsafe impl HasDelayedWork for Device +where + T: drm::Driver, + T::Data: HasDelayedWork, + C: DeviceContext, +{ +} diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs new file mode 100644 index 000000000..74f6ed690 --- /dev/null +++ b/rust/kernel/drm/driver.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM driver core. +//! +//! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) + +use crate::{ + bindings, + device, + drm, + error::to_result, + prelude::*, + sync::aref::ARef, // +}; +use core::ptr::NonNull; + +/// Driver use the GEM memory manager. This should be set for all modern drivers. +pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports render nodes, i.e.: /dev/dri/renderDXX devices. +pub(crate) const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; + +/// Information data for a DRM Driver. +pub struct DriverInfo { + /// Driver major version. + pub major: i32, + /// Driver minor version. + pub minor: i32, + /// Driver patchlevel version. + pub patchlevel: i32, + /// Driver name. + pub name: &'static CStr, + /// Driver description. + pub desc: &'static CStr, +} + +/// Internal memory management operation set, normally created by memory managers (e.g. GEM). +pub struct AllocOps { + pub(crate) gem_create_object: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) prime_handle_to_fd: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + handle: u32, + flags: u32, + prime_fd: *mut core::ffi::c_int, + ) -> core::ffi::c_int, + >, + pub(crate) prime_fd_to_handle: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + prime_fd: core::ffi::c_int, + handle: *mut u32, + ) -> core::ffi::c_int, + >, + pub(crate) gem_prime_import: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + dma_buf: *mut bindings::dma_buf, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) gem_prime_import_sg_table: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + attach: *mut bindings::dma_buf_attachment, + sgt: *mut bindings::sg_table, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) dumb_create: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + args: *mut bindings::drm_mode_create_dumb, + ) -> core::ffi::c_int, + >, + pub(crate) dumb_map_offset: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + handle: u32, + offset: *mut u64, + ) -> core::ffi::c_int, + >, +} + +/// Trait for memory manager implementations. Implemented internally. +pub trait AllocImpl: super::private::Sealed + drm::gem::IntoGEMObject { + /// The [`Driver`] implementation for this [`AllocImpl`]. + type Driver: drm::Driver; + + /// The C callback operations for this memory manager. + const ALLOC_OPS: AllocOps; +} + +/// The DRM `Driver` trait. +/// +/// This trait must be implemented by drivers in order to create a `struct drm_device` and `struct +/// drm_driver` to be registered in the DRM subsystem. +#[vtable] +pub trait Driver { + /// Context data associated with the DRM driver + type Data: Sync + Send; + + /// Data owned by the [`Registration`] and accessible within a + /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via + /// [`Device::registration_data_with()`](drm::Device::registration_data_with). + /// + /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the + /// parent bus device binding scope but may be shorter. + type RegistrationData<'a>: Send + Sync + 'a; + + /// The type used to manage memory for this driver. + type Object: AllocImpl; + + /// The type used to represent a DRM File (client) + type File: drm::file::DriverFile; + + /// The bus device type of the parent device that the DRM device is associated with. + type ParentDevice: device::AsBusDevice; + + /// Driver metadata + const INFO: DriverInfo; + + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; + + /// Sets the `DRIVER_RENDER` feature for this driver. + /// + /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to + /// userspace. The render node is an alternate low-privilege way to access + /// the driver, which is enforced on a per-ioctl level. Userspace processes + /// that open the render node can only invoke ioctls explicitly listed as + /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas + /// userspace processes using the master node can invoke any ioctl. + const FEAT_RENDER: bool = false; +} + +/// The registration type of a `drm::Device`. +/// +/// Once the `Registration` structure is dropped, the device is unregistered. +pub struct Registration<'a, T: Driver> { + drm: ARef>, + _reg_data: Pin>>, +} + +impl<'a, T: Driver> Registration<'a, T> { + /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + pub unsafe fn new( + dev: &'a device::Device, + drm: drm::UnregisteredDevice, + reg_data: impl PinInit, E>, + flags: usize, + ) -> Result + where + Error: From, + { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { + return Err(EINVAL); + } + + let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once ioctls can be called. + let ptr: NonNull> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + if let Err(e) = to_result(ret) { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent + // access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + return Err(e); + } + + Ok(Self { + drm: (&*drm).into(), + _reg_data: reg_data, + }) + } + + /// Returns a reference to the `Device` instance for this registration. + pub fn device(&self) -> &drm::Device { + &self.drm + } +} + +// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between +// threads, hence it's safe to share it. +unsafe impl Sync for Registration<'_, T> {} + +// SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. +unsafe impl Send for Registration<'_, T> {} + +impl Drop for Registration<'_, T> { + fn drop(&mut self) { + // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing + // `drm_dev_enter()` critical sections complete before unregistration proceeds. This + // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in + // `drm_dev_unplug()` to guarantee that the parent device is still bound within the + // critical section. + // + // SAFETY: Safe by the invariant of `ARef>`. The existence of this + // `Registration` also guarantees that this `drm::Device` is actually registered. + unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) }; + // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical + // sections have completed, so no one holds a reference to reg_data anymore. + // reg_data is dropped here automatically. + } +} diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs new file mode 100644 index 000000000..10160601c --- /dev/null +++ b/rust/kernel/drm/file.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM File objects. +//! +//! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) + +use crate::{ + bindings, + drm, + prelude::*, + types::Opaque, // +}; +use core::marker::PhantomData; + +/// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). +pub trait DriverFile { + /// The parent `Driver` implementation for this `DriverFile`. + type Driver: drm::Driver; + + /// Open a new file (called when a client opens the DRM device). + fn open(device: &drm::Device) -> Result>>; +} + +/// An open DRM File. +/// +/// # Invariants +/// +/// `self.0` is a valid instance of a `struct drm_file`. +#[repr(transparent)] +pub struct File(Opaque, PhantomData); + +impl File { + #[doc(hidden)] + /// Not intended to be called externally, except via declare_drm_ioctls!() + /// + /// # Safety + /// + /// `raw_file` must be a valid pointer to an open `struct drm_file`, opened through `T::open`. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_file) -> &'a File { + // SAFETY: `raw_file` is valid by the safety requirements of this function. + unsafe { &*ptr.cast() } + } + + pub(super) fn as_raw(&self) -> *mut bindings::drm_file { + self.0.get() + } + + fn driver_priv(&self) -> *mut T { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. + unsafe { (*self.as_raw()).driver_priv }.cast() + } + + /// Return a pinned reference to the driver file structure. + pub fn inner(&self) -> Pin<&T> { + // SAFETY: By the type invariant the pointer `self.as_raw()` points to a valid and opened + // `struct drm_file`, hence `driver_priv` has been properly initialized by `open_callback`. + unsafe { Pin::new_unchecked(&*(self.driver_priv())) } + } + + /// The open callback of a `struct drm_file`. + pub(crate) extern "C" fn open_callback( + raw_dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, + ) -> core::ffi::c_int { + // SAFETY: A callback from `struct drm_driver::open` guarantees that + // - `raw_dev` is valid pointer to a `struct drm_device`, + // - the corresponding `struct drm_device` has been registered. + let drm = unsafe { drm::Device::from_raw(raw_dev) }; + + // SAFETY: `raw_file` is a valid pointer to a `struct drm_file`. + let file = unsafe { File::::from_raw(raw_file) }; + + let inner = match T::open(drm) { + Err(e) => { + return e.to_errno(); + } + Ok(i) => i, + }; + + // SAFETY: This pointer is treated as pinned, and the Drop guarantee is upheld in + // `postclose_callback()`. + let driver_priv = KBox::into_raw(unsafe { Pin::into_inner_unchecked(inner) }); + + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. + unsafe { (*file.as_raw()).driver_priv = driver_priv.cast() }; + + 0 + } + + /// The postclose callback of a `struct drm_file`. + pub(crate) extern "C" fn postclose_callback( + _raw_dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: This reference won't escape this function + let file = unsafe { File::::from_raw(raw_file) }; + + // SAFETY: `file.driver_priv` has been created in `open_callback` through `KBox::into_raw`. + let _ = unsafe { KBox::from_raw(file.driver_priv()) }; + } +} + +impl super::private::Sealed for File {} diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs new file mode 100644 index 000000000..e1ebad77e --- /dev/null +++ b/rust/kernel/drm/gem/mod.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM GEM API +//! +//! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) + +use crate::{ + bindings, + drm::{ + self, + device::{ + DeviceContext, + Normal, // + }, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::NonNull, // +}; + +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; + +/// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its +/// [`DriverObject`] implementation. +/// +/// [`Driver`]: drm::Driver +/// [`DriverFile`]: drm::file::DriverFile +pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; + +/// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. +/// +/// [`Driver`]: drm::Driver +pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; + +/// GEM object functions, which must be implemented by drivers. +pub trait DriverObject: Sync + Send + Sized + 'static { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + + /// Create a new driver data object for a GEM object of a given size. + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; + + /// Open a new handle to an existing object, associated with a File. + fn open(_obj: &DriverAllocImpl, _file: &DriverFile) -> Result { + Ok(()) + } + + /// Close a handle to an existing object, associated with a File. + fn close(_obj: &DriverAllocImpl, _file: &DriverFile) {} +} + +/// Trait that represents a GEM object subtype +pub trait IntoGEMObject: Sized + super::private::Sealed { + /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as + /// this owning object is valid. + fn as_raw(&self) -> *mut bindings::drm_gem_object; + + /// Converts a pointer to a `struct drm_gem_object` into a reference to `Self`. + /// + /// # Safety + /// + /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a + /// valid instance of `Self`. + /// - The caller promises that holding the immutable reference returned by this function does + /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; +} + +extern "C" fn open_callback( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) -> core::ffi::c_int { + // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. + let file = unsafe { DriverFile::::from_raw(raw_file) }; + + // SAFETY: + // * `open_callback` is specified in the AllocOps structure for `DriverObject`, ensuring that + // `raw_obj` is contained within a `DriverAllocImpl` + // * It is only possible for `open_callback` to be called after device registration, ensuring + // that the object's device is in the `Registered` state. + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; + + match T::open(obj, file) { + Err(e) => e.to_errno(), + Ok(()) => 0, + } +} + +extern "C" fn close_callback( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) { + // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. + let file = unsafe { DriverFile::::from_raw(raw_file) }; + + // SAFETY: `close_callback` is specified in the AllocOps structure for `Object`, ensuring + // that `raw_obj` is indeed contained within a `Object`. + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; + + T::close(obj, file); +} + +impl IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { + // SAFETY: `obj` is guaranteed to be in an `Object` via the safety contract of this + // function + unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object, obj) } + } +} + +/// Base operations shared by all GEM object classes +pub trait BaseObject: IntoGEMObject { + /// Returns the size of the object in bytes. + fn size(&self) -> usize { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `struct drm_gem_object`. + unsafe { (*self.as_raw()).size } + } + + /// Creates a new handle for the object associated with a given `File` + /// (or returns an existing one). + fn create_handle(&self, file: &drm::File) -> Result + where + Self: AllocImpl, + D: drm::Driver, + F: drm::file::DriverFile, + { + let mut handle: u32 = 0; + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_handle_create(file.as_raw().cast(), self.as_raw(), &mut handle) + })?; + Ok(handle) + } + + /// Looks up an object by its handle for a given `File`. + fn lookup_handle(file: &drm::File, handle: u32) -> Result> + where + Self: AllocImpl + AlwaysRefCounted, + D: drm::Driver, + F: drm::file::DriverFile, + { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_gem_object_lookup(file.as_raw().cast(), handle) }; + if ptr.is_null() { + return Err(ENOENT); + } + + // SAFETY: + // - A `drm::Driver` can only have a single `File` implementation. + // - `file` uses the same `drm::Driver` as `Self`. + // - Therefore, we're guaranteed that `ptr` must be a gem object embedded within `Self`. + // - And we check if the pointer is null befoe calling from_raw(), ensuring that `ptr` is a + // valid pointer to an initialized `Self`. + let obj = unsafe { Self::from_raw(ptr) }; + + // SAFETY: + // - We take ownership of the reference of `drm_gem_object_lookup()`. + // - Our `NonNull` comes from an immutable reference, thus ensuring it is a valid pointer to + // `Self`. + Ok(unsafe { ARef::from_raw(obj.into()) }) + } + + /// Creates an mmap offset to map the object from userspace. + fn create_mmap_offset(&self) -> Result { + // SAFETY: The arguments are valid per the type invariant. + to_result(unsafe { bindings::drm_gem_create_mmap_offset(self.as_raw()) })?; + + // SAFETY: The arguments are valid per the type invariant. + Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) + } +} + +impl BaseObject for T {} + +/// Crate-private base operations shared by all GEM object classes. +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + +/// A base GEM object. +/// +/// # Invariants +/// +/// * `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. +#[repr(C)] +#[pin_data] +pub struct Object { + obj: Opaque, + #[pin] + data: T, + _ctx: PhantomData, +} + +impl Object { + const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(open_callback::), + close: Some(close_callback::), + print_info: None, + export: None, + pin: None, + unpin: None, + get_sg_table: None, + vmap: None, + vunmap: None, + mmap: None, + status: None, + vm_ops: core::ptr::null_mut(), + evict: None, + rss: None, + }; + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &drm::Device { + // SAFETY: + // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM + // object lives. + // - The device we used for creating the gem object is passed as &drm::Device to + // Object::::new(), so we know that `T::Driver` is the right generic parameter to use + // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. + unsafe { drm::Device::from_raw((*self.as_raw()).dev) } + } + + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + let ptr: *mut Opaque = obj.cast(); + + // SAFETY: All of our objects are of type `Object`. + let this = unsafe { crate::container_of!(ptr, Self, obj) }; + + // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct + // drm_gem_object`. + unsafe { bindings::drm_gem_object_release(obj) }; + + // SAFETY: All of our objects are allocated via `KBox`, and we're in the + // free callback which guarantees this object has zero remaining references, + // so we can drop it. + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl Object { + /// Create a new GEM object. + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { + let obj: Pin> = KBox::pin_init( + try_pin_init!(Self { + obj: Opaque::new(bindings::drm_gem_object::default()), + data <- T::new(dev, size, args), + _ctx: PhantomData, + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; + + // INVARIANT: `dev` and the GEM object are in the same state at the moment, and upgrading + // the typestate in `dev` will not carry over to the GEM object. + if let Err(err) = + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) + }) + { + // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before + // failing, so `drm_gem_private_object_fini()` is the matching cleanup. + unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) }; + return Err(err); + } + + // SAFETY: We will never move out of `Self` as `ARef` is always treated as pinned. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); + + // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. + Ok(unsafe { ARef::from_raw(ptr) }) + } +} + +impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject +} + +impl super::private::Sealed for Object {} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: AllocOps = AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: None, + dumb_create: None, + dumb_map_offset: None, + }; +} + +pub(super) const fn create_fops(owner: *mut bindings::module) -> bindings::file_operations { + let mut fops: bindings::file_operations = pin_init::zeroed(); + + fops.owner = owner; + fops.open = Some(bindings::drm_open); + fops.release = Some(bindings::drm_release); + fops.unlocked_ioctl = Some(bindings::drm_ioctl); + #[cfg(CONFIG_COMPAT)] + { + fops.compat_ioctl = Some(bindings::drm_compat_ioctl); + } + fops.poll = Some(bindings::drm_poll); + fops.read = Some(bindings::drm_read); + fops.llseek = Some(bindings::noop_llseek); + fops.mmap = Some(bindings::drm_gem_mmap); + fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET; + + fops +} diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 000000000..a687d46d1 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + device::{ + self, + Bound, // + }, + devres::*, + drm::{ + driver, + gem, + private::Sealed, + Device, // + }, + error::{ + from_err_ptr, + to_result, // + }, + io::{ + IoBase, + Region, + SysMem, + SysMemBackend, // + }, + prelude::*, + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, +}; +use core::{ + ffi::c_void, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::{ + self, + NonNull, // + }, +}; +use gem::{ + BaseObject, + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject +} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Send for Object {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within an + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> + where + R: Deref + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap(&self) -> Result> { + self.make_vmap() + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { + let parent = self.dev().as_ref(); + if dev.as_raw() != parent.as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap(&self) -> Result> { + self.make_vmap() + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Self, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject>(&'a Object, NotThreadSafe); + +impl<'a, T: DriverObject> DmaResvGuard<'a, T> { + #[inline] + fn new(obj: &'a Object) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap +where + D: DriverObject, + R: Deref>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned = VMap>, SIZE>; + +impl VMap +where + D: DriverObject, + R: Deref>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl<'a, D, R, const SIZE: usize> IoBase<'a> for &'a VMap +where + D: DriverObject, + R: Deref>, +{ + type Backend = SysMemBackend; + type Target = Region; + + #[inline] + fn as_view(self) -> SysMem<'a, Region> { + let ptr = Region::ptr_from_raw_parts_mut(self.addr.cast(), self.owner.size()); + + // SAFETY: Per type invariants of `VMap`: + // - `addr .. addr + owner.size()` is a valid kernel accessible memory region. + // - `addr` is page-aligned, which satisfies `Region`'s 4-byte alignment requirement. + // - The memory remains valid until this `VMap` is dropped; since `self` is `&'a VMap`, + // the borrow prevents the `VMap` from being dropped for the lifetime `'a`. + unsafe { SysMem::new(ptr) } + } +} + +impl Drop for VMap +where + D: DriverObject, + R: Deref>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl Send for VMap +where + D: DriverObject, + R: Deref> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl Sync for VMap +where + D: DriverObject, + R: Deref> + Sync, +{ +} + +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, +} + +impl Deref for SGTableMap { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl Drop for SGTableMap { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + io::Io, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device, + _size: usize, + _args: Self::Args, + ) -> impl PinInit { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = KunitFile; + type Object = Object; + type ParentDevice = faux::Device; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let reg = faux::Registration::new(c"Kunit", None)?; + let fdev = reg.as_ref(); + let drm = UnregisteredDevice::new(fdev, data)?; + + Ok((reg, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the size matches the actual object size + assert_eq!(vmap.size(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.try_read8(offset).unwrap(), expected); + } + + Ok(()) + } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"EvilKunit", None)?; + let wrong_dev = reg.as_ref(); + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } +} diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs new file mode 100644 index 000000000..d9d43d719 --- /dev/null +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![cfg(CONFIG_RUST_DRM_GPUVM)] + +//! DRM GPUVM in immediate mode +//! +//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated +//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM +//! and the GPU's virtual address space has the same state at all times. +//! +//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) + +use kernel::{ + alloc::{ + AllocError, + Flags as AllocFlags, // + }, + bindings, + drm, + drm::gem::IntoGEMObject, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Range, // + }, + ptr::{ + self, + NonNull, // + }, // +}; + +mod sm_ops; +pub use self::sm_ops::*; + +mod vm_bo; +pub use self::vm_bo::*; + +mod va; +pub use self::va::*; + +/// A DRM GPU VA manager. +/// +/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed +/// via the special unique handle [`UniqueRefGpuVm`]. +/// +/// # Invariants +/// +/// * Stored in an allocation managed by the refcount in `self.vm`. +/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. +/// * Does not contain any sparse [`GpuVa`] instances. +#[pin_data] +pub struct GpuVm { + #[pin] + vm: Opaque, + /// Accessed only through the [`UniqueRefGpuVm`] reference. + data: UnsafeCell, +} + +// SAFETY: It is safe to send a `GpuVm` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. +unsafe impl Send for GpuVm {} +// SAFETY: It is safe to share a `&GpuVm` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. +unsafe impl Sync for GpuVm {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; + } +} + +impl PartialEq for GpuVm { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVm {} + +impl GpuVm { + const fn vtable() -> &'static bindings::drm_gpuvm_ops { + &bindings::drm_gpuvm_ops { + vm_free: Some(Self::vm_free), + op_alloc: None, + op_free: None, + vm_bo_alloc: GpuVmBo::::ALLOC_FN, + vm_bo_free: GpuVmBo::::FREE_FN, + vm_bo_validate: None, + sm_step_map: Some(Self::sm_step_map), + sm_step_unmap: Some(Self::sm_step_unmap), + sm_step_remap: Some(Self::sm_step_remap), + } + } + + /// Creates a GPUVM instance. + #[expect(clippy::new_ret_no_self)] + pub fn new( + name: &'static CStr, + dev: &drm::Device, + r_obj: &T::Object, + range: Range, + reserve_range: Range, + data: T, + ) -> Result, E> + where + E: From, + E: From, + { + let obj = KBox::try_pin_init::( + try_pin_init!(Self { + data: UnsafeCell::new(data), + vm <- Opaque::ffi_init(|vm| { + // SAFETY: These arguments are valid. `vm` is valid until refcount drops to + // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag + // below. + unsafe { + bindings::drm_gpuvm_init( + vm, + name.as_char_ptr(), + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, + dev.as_raw(), + r_obj.as_raw(), + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + const { Self::vtable() }, + ) + } + }), + }? E), + GFP_KERNEL | __GFP_ZERO, + )?; + // SAFETY: This transfers the initial refcount to the ARef. + let aref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( + Pin::into_inner_unchecked(obj), + ))) + }; + // INVARIANT: This reference is unique. + Ok(UniqueRefGpuVm(aref)) + } + + /// Access this [`GpuVm`] from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm`] that remains + /// valid for at least `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. Caller ensures the + // pointer is valid for 'a. + unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } + } + + /// Returns a raw pointer to the embedded `struct drm_gpuvm`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { + self.vm.get() + } + + /// The start of the VA space. + #[inline] + pub fn va_start(&self) -> u64 { + // SAFETY: The `mm_start` field is immutable. + unsafe { (*self.as_raw()).mm_start } + } + + /// The length of the GPU's virtual address space. + #[inline] + pub fn va_length(&self) -> u64 { + // SAFETY: The `mm_range` field is immutable. + unsafe { (*self.as_raw()).mm_range } + } + + /// Returns the range of the GPU virtual address space. + #[inline] + pub fn va_range(&self) -> Range { + let start = self.va_start(); + // OVERFLOW: This reconstructs the Range passed to the constructor, so it won't fail. + let end = start + self.va_length(); + Range { start, end } + } + + /// Get or create the [`GpuVmBo`] for this gem object. + #[inline] + pub fn obtain( + &self, + obj: &T::Object, + data: impl PinInit, + ) -> Result>, AllocError> { + Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) + } + + /// Clean up buffer objects that are no longer used. + #[inline] + pub fn deferred_cleanup(&self) { + // SAFETY: This GPUVM uses immediate mode. + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } + } + + /// Check if this GEM object is an external object for this GPUVM. + #[inline] + pub fn is_extobj(&self, obj: &T::Object) -> bool { + // SAFETY: We may call this with any GPUVM and GEM object. + unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } + } + + /// Free this GPUVM. + /// + /// # Safety + /// + /// Called when refcount hits zero. + unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. + let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; + // SAFETY: By type invariants we can free it when refcount hits zero. + drop(unsafe { KBox::from_raw(me) }) + } + + #[inline] + fn raw_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. + unsafe { (*(*self.as_raw()).r_obj).resv } + } +} + +/// The manager for a GPUVM. +pub trait DriverGpuVm: Sized + Send + Sync { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The kind of GEM object stored in this GPUVM. + type Object: drm::driver::AllocImpl + Send + Sync; + + /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; + + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). + type VmBoData: Send + Sync; + + /// The private data passed to callbacks. + type SmContext<'ctx> + where + Self: 'ctx; + + /// Indicates that a new mapping should be created. + fn sm_step_map<'op, 'ctx>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op, 'ctx>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + + /// Indicates that an existing mapping should be split up. + fn sm_step_remap<'op, 'ctx>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; +} + +/// The core of the DRM GPU VA manager. +/// +/// This object is a unique reference to the VM that can access the interval tree and the Rust +/// `data` field. +/// +/// # Invariants +/// +/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. +pub struct UniqueRefGpuVm(ARef>); + +impl UniqueRefGpuVm { + /// Access the data owned by this `UniqueRefGpuVm` immutably. + #[inline] + pub fn data_ref(&self) -> &T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &*self.0.data.get() } + } + + /// Access the data owned by this `UniqueRefGpuVm` mutably. + #[inline] + pub fn data(&mut self) -> &mut T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &mut *self.0.data.get() } + } +} + +impl Deref for UniqueRefGpuVm { + type Target = GpuVm; + + #[inline] + fn deref(&self) -> &GpuVm { + &self.0 + } +} diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs new file mode 100644 index 000000000..742c151b2 --- /dev/null +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// The actual data that gets threaded through the callbacks. +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { + gpuvm: &'a mut UniqueRefGpuVm, + user_context: &'a mut T::SmContext<'ctx>, +} + +/// Adds an extra field to `SmData` for `sm_map()` callbacks. +/// +/// # Invariants +/// +/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`. +#[repr(C)] +struct SmMapData<'a, 'ctx, T: DriverGpuVm> { + sm_data: SmData<'a, 'ctx, T>, + vm_bo: &'a GpuVmBo, +} + +/// The argument for [`UniqueRefGpuVm::sm_map`]. +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { + /// Address in GPU virtual address space. + pub addr: u64, + /// Length of mapping to create. + pub range: u64, + /// Offset in GEM object. + pub gem_offset: u64, + /// The GEM object to map. + pub vm_bo: &'a GpuVmBo, + /// The user-provided context type. + pub context: &'a mut T::SmContext<'ctx>, +} + +impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> { + fn raw_request(&self) -> bindings::drm_gpuvm_map_req { + bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: self.addr, + range: self.range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: self.gem_offset, + obj: self.vm_bo.obj().as_raw(), + }, + }, + } + } +} + +/// Represents an `sm_step_map` operation that has not yet been completed. +pub struct OpMap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_map, + // Since these abstractions are designed for immediate mode, the VM BO needs to be + // pre-allocated, so we always have it available when we reach this point. + vm_bo: &'op GpuVmBo, + // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not + // coerce to `OpMap<'short, T>`. This ensures that the user can't return + // the wrong `OpMapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpMap<'op, T> { + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.op.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.op.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.op.gem.offset + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) to map. + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw(self.op.gem.obj) } + } + + /// The [`GpuVmBo`] that the new VA will be associated with. + pub fn vm_bo(&self) -> &GpuVmBo { + self.vm_bo + } + + /// Use the pre-allocated VA to carry out this map operation. + pub fn insert(self, va: GpuVaAlloc, va_data: impl PinInit) -> OpMapped<'op, T> { + let va = va.prepare(va_data); + // SAFETY: By the type invariants we may access the interval tree. + unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) }; + + let _gpuva_guard = self.vm_bo().lock_gpuva(); + // SAFETY: The va is prepared for insertion, and we hold the GEM lock. + unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) }; + + OpMapped { + _invariant: self._invariant, + } + } +} + +/// Represents a completed [`OpMap`] operation. +pub struct OpMapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_unmap` operation that has not yet been completed. +pub struct OpUnmap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_unmap, + // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not + // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the + // wrong`OpUnmapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpUnmap<'op, T> { + /// Indicates whether this [`GpuVa`] is physically contiguous with the + /// original mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table + /// mappings for this `drm_gpuva`, adding the missing page table entries + /// only and update the `drm_gpuvm` accordingly. + pub fn keep(&self) -> bool { + self.op.keep + } + + /// The range being unmapped. + pub fn va(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw(self.op.va) } + } + + /// Remove the VA. + pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved) { + // SAFETY: The op references a valid drm_gpuva in the GPUVM. + unsafe { bindings::drm_gpuva_unmap(self.op) }; + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) }; + + // SAFETY: We just removed this va from the `GpuVm`. + let va = unsafe { GpuVaRemoved::from_raw(self.op.va) }; + + ( + OpUnmapped { + _invariant: self._invariant, + }, + va, + ) + } +} + +/// Represents a completed [`OpUnmap`] operation. +pub struct OpUnmapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_remap` operation that has not yet been completed. +pub struct OpRemap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_remap, + // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not + // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the + // wrong`OpRemapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpRemap<'op, T> { + /// The preceding part of a split mapping. + #[inline] + pub fn prev(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// The subsequent part of a split mapping. + #[inline] + pub fn next(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original + /// mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this + /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm` + /// accordingly. + #[inline] + pub fn keep(&self) -> bool { + // SAFETY: The unmap pointer is always valid. + unsafe { (*self.op.unmap).keep } + } + + /// The range being unmapped. + #[inline] + pub fn va_to_unmap(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw((*self.op.unmap).va) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped. + #[inline] + pub fn obj(&self) -> &T::Object { + self.va_to_unmap().obj() + } + + /// The [`GpuVmBo`] that is being remapped. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + self.va_to_unmap().vm_bo() + } + + /// Update the GPUVM to perform the remapping. + pub fn remap( + self, + va_alloc: [GpuVaAlloc; 2], + prev_data: impl PinInit, + next_data: impl PinInit, + ) -> (OpRemapped<'op, T>, OpRemapRet) { + let [va1, va2] = va_alloc; + + let mut unused_va = None; + let mut prev_ptr = ptr::null_mut(); + let mut next_ptr = ptr::null_mut(); + if self.prev().is_some() { + prev_ptr = va1.prepare(prev_data); + } else { + unused_va = Some(va1); + } + if self.next().is_some() { + next_ptr = va2.prepare(next_data); + } else { + unused_va = Some(va2); + } + + // SAFETY: the pointers are non-null when required + unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) }; + + let gpuva_guard = self.vm_bo().lock_gpuva(); + if !prev_ptr.is_null() { + // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) }; + } + if !next_ptr.is_null() { + // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) }; + } + drop(gpuva_guard); + + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) }; + + ( + OpRemapped { + _invariant: self._invariant, + }, + OpRemapRet { + // SAFETY: We just removed this va from the `GpuVm`. + unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) }, + unused_va, + }, + ) + } +} + +/// Part of an [`OpRemap`] that represents a new mapping. +#[repr(transparent)] +pub struct OpRemapMapData(bindings::drm_gpuva_op_map); + +impl OpRemapMapData { + /// # Safety + /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`. + unsafe fn from_raw<'a>(ptr: NonNull) -> &'a Self { + // SAFETY: ok per safety requirements + unsafe { ptr.cast().as_ref() } + } + + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.0.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.0.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.0.gem.offset + } +} + +/// Struct containing objects removed or not used by [`OpRemap::remap`]. +pub struct OpRemapRet { + /// The `drm_gpuva` that was removed. + pub unmapped_va: GpuVaRemoved, + /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is + /// returned here. + pub unused_va: Option>, +} + +/// Represents a completed [`OpRemap`] operation. +pub struct OpRemapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +impl UniqueRefGpuVm { + /// Create a mapping, removing or remapping anything that overlaps. + /// + /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that + /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping. + #[inline] + pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result { + if req.vm_bo.gpuvm() != &**self { + return Err(EINVAL); + } + + let gpuvm = self.as_raw(); + let raw_req = req.raw_request(); + // INVARIANT: Checked above that `vm_bo.gpuvm() == self`. + let mut p = SmMapData { + sm_data: SmData { + gpuvm: self, + user_context: req.context, + }, + vm_bo: req.vm_bo, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is valid to be interpreted as both SmData and SmMapData since the + // first field of SmMapData is SmData. + to_result(unsafe { + bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req) + }) + } + + /// Remove any mappings in the given region. + /// + /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the + /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range. + #[inline] + pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result { + let gpuvm = self.as_raw(); + let mut p = SmData { + gpuvm: self, + user_context: context, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is a valid SmData. + to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) + } +} + +impl GpuVm { + /// # Safety + /// Must be called from `sm_map` with a pointer to `SmMapData`. + pub(super) unsafe extern "C" fn sm_step_map( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes + // an `SmMapData` as private data. + let p = unsafe { &mut *p.cast::>() }; + let op = OpMap { + // SAFETY: sm_step_map is called with a map operation. + op: unsafe { &(*op).__bindgen_anon_1.map }, + vm_bo: p.vm_bo, + _invariant: PhantomData, + }; + match p + .sm_data + .gpuvm + .data() + .sm_step_map(op, p.sm_data.user_context) + { + Ok(OpMapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_unmap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpUnmap { + // SAFETY: sm_step_unmap is called with an unmap operation. + op: unsafe { &(*op).__bindgen_anon_1.unmap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_unmap(op, p.user_context) { + Ok(OpUnmapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_remap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpRemap { + // SAFETY: sm_step_remap is called with a remap operation. + op: unsafe { &(*op).__bindgen_anon_1.remap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_remap(op, p.user_context) { + Ok(OpRemapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } +} diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs new file mode 100644 index 000000000..46493f0ec --- /dev/null +++ b/rust/kernel/drm/gpuvm/va.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * This is a valid `drm_gpuva` object that is resident in a [`GpuVm`] instance. +/// * It is associated with a [`GpuVmBo`]. Or in other words, it's not an +/// `gpuvm->kernel_alloc_node` and `DRM_GPUVA_SPARSE` is not set. +/// * The associated [`GpuVmBo`] is part of the GEM list. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + inner: Opaque, + #[pin] + data: T::VaData, +} + +impl PartialEq for GpuVa { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVa {} + +impl GpuVa { + /// Access this [`GpuVa`] from a raw pointer. + /// + /// # Safety + /// + /// * For the duration of `'a`, the pointer must reference a valid `drm_gpuva` associated with + /// a [`GpuVm`]. + /// * It must be associated with a [`GpuVmBo`]. + /// * The associated [`GpuVmBo`] is part of the GEM list. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuva) -> &'a Self { + // CAST: `drm_gpuva` is first field and `repr(C)`. + // SAFETY: The safety requirements match the invariants of `GpuVa`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuva { + self.inner.get() + } + + /// Returns the address of this mapping in the GPU virtual address space. + #[inline] + pub fn addr(&self) -> u64 { + // SAFETY: The `va.addr` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.addr } + } + + /// Returns the length of this mapping. + #[inline] + pub fn length(&self) -> u64 { + // SAFETY: The `va.range` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.range } + } + + /// Returns `addr..addr+length`. + #[inline] + pub fn range(&self) -> Range { + let addr = self.addr(); + addr..addr + self.length() + } + + /// Returns the offset within the GEM object. + #[inline] + pub fn gem_offset(&self) -> u64 { + // SAFETY: The `gem.offset` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).gem.offset } + } + + /// Returns the GEM object. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `gem.obj` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. + unsafe { ::from_raw((*self.as_raw()).gem.obj) } + } + + /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`]. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + // SAFETY: The `vm_bo` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. The BO is in the GEM list by the type + // invariants. + unsafe { GpuVmBo::from_raw((*self.as_raw()).vm_bo) } + } +} + +/// A pre-allocated [`GpuVa`] object. +/// +/// # Invariants +/// +/// The memory is zeroed. +pub struct GpuVaAlloc(KBox>>); + +// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no +// thread-bound state. +unsafe impl Send for GpuVaAlloc {} + +// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared +// `&GpuVaAlloc` cannot access the allocation. +unsafe impl Sync for GpuVaAlloc {} + +impl GpuVaAlloc { + /// Pre-allocate a [`GpuVa`] object. + pub fn new(flags: AllocFlags) -> Result, AllocError> { + // INVARIANTS: Memory allocated with __GFP_ZERO. + Ok(GpuVaAlloc(KBox::new_uninit(flags | __GFP_ZERO)?)) + } + + /// Prepare this `drm_gpuva` for insertion into the GPUVM. + #[must_use] + pub(super) fn prepare(mut self, va_data: impl PinInit) -> *mut bindings::drm_gpuva { + let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); + // SAFETY: The `data` field is pinned. + unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) }; + KBox::into_raw(self.0).cast() + } +} + +/// A [`GpuVa`] object that has been removed. +/// +/// # Invariants +/// +/// The `drm_gpuva` is not resident in the [`GpuVm`]. +pub struct GpuVaRemoved(KBox>); + +impl GpuVaRemoved { + /// Convert a raw pointer into a [`GpuVaRemoved`]. + /// + /// # Safety + /// + /// * Must have been removed from a [`GpuVm`]. + /// * It must not be a `gpuvm->kernel_alloc_node` va. + pub(super) unsafe fn from_raw(ptr: *mut bindings::drm_gpuva) -> Self { + // SAFETY: Since it used to be a VA in a `GpuVm` and it's not a kernel_alloc_node, this + // pointer references a `GpuVa` with a valid `T::VaData`. Since it has been removed, we + // can take ownership of the allocation. + GpuVaRemoved(unsafe { KBox::from_raw(ptr.cast()) }) + } + + /// Take ownership of the VA data. + pub fn into_inner(self) -> T::VaData + where + T::VaData: Unpin, + { + KBox::into_inner(self.0).data + } +} + +impl Deref for GpuVaRemoved { + type Target = T::VaData; + fn deref(&self) -> &T::VaData { + &self.0.data + } +} + +impl DerefMut for GpuVaRemoved +where + T::VaData: Unpin, +{ + fn deref_mut(&mut self) -> &mut T::VaData { + &mut self.0.data + } +} diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs new file mode 100644 index 000000000..5989972da --- /dev/null +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * Allocated with `kmalloc` and refcounted via `inner`. +/// * Is present in the gem list. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + inner: Opaque, + #[pin] + data: T::VmBoData, +} + +// SAFETY: It is safe to send a `GpuVmBo` to another thread: dropping it there drops +// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds. +unsafe impl Send for GpuVmBo {} + +// SAFETY: It is safe to share a `&GpuVmBo` between threads: it effectively shares +// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an +// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds. +unsafe impl Sync for GpuVmBo {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // CAST: `drm_gpuvm_bo` is first field of repr(C) struct. + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred + // mechanism. + unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) }; + } +} + +impl PartialEq for GpuVmBo { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVmBo {} + +impl GpuVmBo { + /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable. + /// + /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to + /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`. + /// + /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa. + pub(super) const ALLOC_FN: Option *mut bindings::drm_gpuvm_bo> = { + use core::alloc::Layout; + let base = Layout::new::(); + let rust = Layout::new::(); + assert!(base.size() <= rust.size()); + if base.size() != rust.size() || base.align() != rust.align() { + Some(Self::vm_bo_alloc) + } else { + // This causes GPUVM to allocate a `GpuVmBo` with `kzalloc(sizeof(drm_gpuvm_bo))`. + None + } + }; + + /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable. + /// + /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if + /// the default gpuvm behavior is the same as `vm_bo_free`. + /// + /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa. + pub(super) const FREE_FN: Option = { + if core::mem::needs_drop::() { + Some(Self::vm_bo_free) + } else { + // This causes GPUVM to free a `GpuVmBo` with `kfree`. + None + } + }; + + /// Custom function for allocating a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// Always safe to call. + unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo { + let raw_ptr = KBox::::new_uninit(GFP_KERNEL | __GFP_ZERO) + .map(KBox::into_raw) + .unwrap_or(ptr::null_mut()); + + // CAST: `drm_gpuvm_bo` is first field of `Self`. + raw_ptr.cast() + } + + /// Custom function for freeing a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after + /// this call. + unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) { + // CAST: `drm_gpuvm_bo` is first field of `Self`. + // SAFETY: + // * The ptr was allocated from kmalloc with the layout of `GpuVmBo`. + // * `ptr->inner` has no destructor. + // * `ptr->data` contains a valid `T::VmBoData` that we can drop. + drop(unsafe { KBox::::from_raw(ptr.cast()) }); + } + + /// Access this [`GpuVmBo`] from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with + /// a [`GpuVm`]. The BO must also be present in the GEM list. + #[inline] + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self { + // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + self.inner.get() + } + + /// The [`GpuVm`] that this GEM object is mapped in. + #[inline] + pub fn gpuvm(&self) -> &GpuVm { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { GpuVm::::from_raw((*self.inner.get()).vm) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw((*self.inner.get()).obj) } + } + + /// The driver data with this buffer object. + #[inline] + pub fn data(&self) -> &T::VmBoData { + &self.data + } + + pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> { + // SAFETY: The GEM object is valid. + let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock }; + // SAFETY: The GEM object is valid, so the mutex is properly initialized. + let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) }; + mutex.lock() + } +} + +/// A pre-allocated [`GpuVmBo`] object. +/// +/// # Invariants +/// +/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is +/// absent from any gem, extobj, or evict lists. +pub(super) struct GpuVmBoAlloc(NonNull>); + +impl GpuVmBoAlloc { + /// Create a new pre-allocated [`GpuVmBo`]. + /// + /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call + /// drop on the data, so we don't have a way to free it when the data is missing. + #[inline] + pub(super) fn new( + gpuvm: &GpuVm, + gem: &T::Object, + value: impl PinInit, + ) -> Result, AllocError> { + // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout + // of `GpuVmBo`. The type is repr(C), so `container_of` is not required. + // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call. + let raw_ptr = unsafe { + bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::>() + }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + // SAFETY: `ptr->data` is a valid pinned location. + unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) }; + // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid + // as we just initialized it. + Ok(GpuVmBoAlloc(ptr)) + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + // SAFETY: The pointer references a valid `drm_gpuvm_bo`. + unsafe { (*self.0.as_ptr()).inner.get() } + } + + /// Look up whether there is an existing [`GpuVmBo`] for this gem object. + /// + /// The caller should not hold the GEM mutex or DMA resv lock. + #[inline] + pub(super) fn obtain(self) -> ARef> { + let me = ManuallyDrop::new(self); + // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this + // call. + let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) }; + + // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr + let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list. + // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`. + let ret = unsafe { ARef::>::from_raw(nonnull) }; + + // Ensure that external objects are in the extobj list. + // + // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where + // we could end up using the extobj before the thread with `ptr == me` calls extobj_add. + if ret.gpuvm().is_extobj(ret.obj()) { + let resv_lock = ret.gpuvm().raw_resv(); + // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists. + // SAFETY: The GPUVM is still alive, so its resv lock is too. + unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) }; + // SAFETY: We hold the GPUVMs resv lock. + unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) }; + // SAFETY: We took the lock, so we can unlock it. + unsafe { bindings::dma_resv_unlock(resv_lock) }; + } + + ret + } +} + +impl Deref for GpuVmBoAlloc { + type Target = GpuVmBo; + #[inline] + fn deref(&self) -> &GpuVmBo { + // SAFETY: By the type invariants we may deref while `Self` exists. + unsafe { self.0.as_ref() } + } +} + +impl Drop for GpuVmBoAlloc { + #[inline] + fn drop(&mut self) { + // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly. + // SAFETY: It's safe to perform a deferred put in any context. + unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) }; + } +} diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs new file mode 100644 index 000000000..64af9eacc --- /dev/null +++ b/rust/kernel/drm/ioctl.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM IOCTL definitions. +//! +//! C header: [`include/drm/drm_ioctl.h`](srctree/include/drm/drm_ioctl.h) + +use crate::ioctl; + +const BASE: u32 = uapi::DRM_IOCTL_BASE as u32; + +/// Construct a DRM ioctl number with no argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IO(nr: u32) -> u32 { + ioctl::_IO(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-only argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOR(nr: u32) -> u32 { + ioctl::_IOR::(BASE, nr) +} + +/// Construct a DRM ioctl number with a write-only argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOW(nr: u32) -> u32 { + ioctl::_IOW::(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-write argument. +#[allow(non_snake_case)] +#[inline(always)] +pub const fn IOWR(nr: u32) -> u32 { + ioctl::_IOWR::(BASE, nr) +} + +/// Descriptor type for DRM ioctls. Use the `declare_drm_ioctls!{}` macro to construct them. +pub type DrmIoctlDescriptor = bindings::drm_ioctl_desc; + +/// This is for ioctl which are used for rendering, and require that the file descriptor is either +/// for a render node, or if it’s a legacy/primary node, then it must be authenticated. +pub const AUTH: u32 = bindings::drm_ioctl_flags_DRM_AUTH; + +/// This must be set for any ioctl which can change the modeset or display state. Userspace must +/// call the ioctl through a primary node, while it is the active master. +/// +/// Note that read-only modeset ioctl can also be called by unauthenticated clients, or when a +/// master is not the currently active one. +pub const MASTER: u32 = bindings::drm_ioctl_flags_DRM_MASTER; + +/// Anything that could potentially wreak a master file descriptor needs to have this flag set. +/// +/// Current that’s only for the SETMASTER and DROPMASTER ioctl, which e.g. logind can call to +/// force a non-behaving master (display compositor) into compliance. +/// +/// This is equivalent to callers with the SYSADMIN capability. +pub const ROOT_ONLY: u32 = bindings::drm_ioctl_flags_DRM_ROOT_ONLY; + +/// This is used for all ioctl needed for rendering only, for drivers which support render nodes. +/// This should be all new render drivers, and hence it should be always set for any ioctl with +/// `AUTH` set. Note though that read-only query ioctl might have this set, but have not set +/// DRM_AUTH because they do not require authentication. +pub const RENDER_ALLOW: u32 = bindings::drm_ioctl_flags_DRM_RENDER_ALLOW; + +/// Internal structures used by the `declare_drm_ioctls!{}` macro. Do not use directly. +#[doc(hidden)] +pub mod internal { + pub use bindings::drm_device; + pub use bindings::drm_file; + pub use bindings::drm_ioctl_desc; + + /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type + /// parameter `T`. + /// + /// Used by [`declare_drm_ioctls!`] to anchor type inference. + #[doc(hidden)] + #[inline] + pub const fn __dev_ctx_cast( + ptr: *const crate::drm::Device, + ) -> *const crate::drm::Device { + ptr.cast() + } +} + +/// Declare the DRM ioctls for a driver. +/// +/// Each entry in the list should have the form: +/// +/// `(ioctl_number, argument_type, flags, user_callback),` +/// +/// `argument_type` is the type name within the `bindings` crate. +/// `user_callback` should have the following prototype: +/// +/// ```ignore +/// fn foo(device: &kernel::drm::Device, +/// reg_data: &Self::RegistrationData<'_>, +/// data: &mut uapi::argument_type, +/// file: &kernel::drm::File, +/// ) -> Result +/// ``` +/// where `Self` is the drm::drv::Driver implementation these ioctls are being declared within. +/// +/// # Examples +/// +/// ```ignore +/// kernel::declare_drm_ioctls! { +/// (FOO_GET_PARAM, drm_foo_get_param, ioctl::RENDER_ALLOW, my_get_param_handler), +/// } +/// ``` +/// +#[macro_export] +macro_rules! declare_drm_ioctls { + ( $(($cmd:ident, $struct:ident, $flags:expr, $func:expr)),* $(,)? ) => { + const IOCTLS: &'static [$crate::drm::ioctl::DrmIoctlDescriptor] = { + use $crate::uapi::*; + const _:() = { + let i: u32 = $crate::uapi::DRM_COMMAND_BASE; + // Assert that all the IOCTLs are in the right order and there are no gaps, + // and that the size of the specified type is correct. + $( + let cmd: u32 = $crate::macros::concat_idents!(DRM_IOCTL_, $cmd); + ::core::assert!(i == $crate::ioctl::_IOC_NR(cmd)); + ::core::assert!(core::mem::size_of::<$crate::uapi::$struct>() == + $crate::ioctl::_IOC_SIZE(cmd)); + let i: u32 = i + 1; + )* + }; + + let ioctls = &[$( + $crate::drm::ioctl::internal::drm_ioctl_desc { + cmd: $crate::macros::concat_idents!(DRM_IOCTL_, $cmd) as u32, + func: { + #[allow(non_snake_case)] + unsafe extern "C" fn $cmd( + raw_dev: *mut $crate::drm::ioctl::internal::drm_device, + raw_data: *mut ::core::ffi::c_void, + raw_file: *mut $crate::drm::ioctl::internal::drm_file, + ) -> core::ffi::c_int { + // SAFETY: + // - The DRM core ensures the device lives while callbacks are being + // called. + // - The DRM device must have been registered when we're called through + // an IOCTL. + // + // INVARIANT: The `Ioctl` context requires that the device has been + // registered via `drm_dev_register()` at some point; the DRM core + // guarantees this for ioctl dispatch callbacks. + // + // FIXME: Currently there is nothing enforcing that the types of the + // dev/file match the current driver these ioctls are being declared + // for, and it's not clear how to enforce this within the type system. + let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> = + $crate::drm::device::Device::from_raw(raw_dev); + + // Type-inference anchor: the closure is never called but ties `dev`'s + // type to `$func`'s first parameter, which the compiler cannot infer + // through method resolution and associated-type projections alone. + #[allow(unreachable_code)] + let _ = || { + let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast( + ::core::ptr::from_ref(dev), + ); + + $func( + // SAFETY: This closure is never executed; the dereference + // exists purely to unify the type parameter with `$func`. + // The pointer is valid regardless. + unsafe { &*__ptr }, + unreachable!(), + unreachable!(), + unreachable!(), + ) + }; + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func; + + let Some(guard) = dev.registration_guard() else { + return $crate::error::code::ENODEV.to_errno(); + }; + + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we + // asserted above matches the size of this type, and all bit patterns of + // UAPI structs must be valid. + // The `ioctl` argument is exclusively owned by the handler + // and guaranteed by the C implementation (`drm_ioctl()`) to remain + // valid for the entire lifetime of the reference taken here. + // There is no concurrent access or aliasing; no other references + // to this object exist during this call. + let data = unsafe { &mut *(raw_data.cast::<$crate::uapi::$struct>()) }; + // SAFETY: This is just the DRM file structure + let file = unsafe { $crate::drm::File::from_raw(raw_file) }; + + match guard.registration_data_with(|reg_data| { + $func(&*guard, reg_data, data, file) + }) { + Err(e) => e.to_errno(), + Ok(i) => i.try_into() + .unwrap_or($crate::error::code::ERANGE.to_errno()), + } + } + Some($cmd) + }, + flags: $flags, + name: $crate::str::as_char_ptr_in_const_context( + $crate::c_str!(::core::stringify!($cmd)), + ), + } + ),*]; + ioctls + }; + }; +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs new file mode 100644 index 000000000..fd6ed35bc --- /dev/null +++ b/rust/kernel/drm/mod.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM subsystem abstractions. + +pub mod device; +pub mod driver; +pub mod file; +pub mod gem; +pub mod gpuvm; +pub mod ioctl; + +pub use self::device::Device; +pub use self::device::DeviceContext; +pub use self::device::Ioctl; +pub use self::device::Normal; +pub use self::device::Registered; +pub use self::device::RegistrationGuard; +pub use self::device::UnregisteredDevice; +pub use self::driver::Driver; +pub use self::driver::DriverInfo; +pub use self::driver::Registration; +pub use self::file::File; + +pub(crate) mod private { + pub trait Sealed {} +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs new file mode 100644 index 000000000..e52793f77 --- /dev/null +++ b/rust/kernel/error.rs @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel errors. +//! +//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)\ +//! C header: [`include/uapi/asm-generic/errno.h`](srctree/include/uapi/asm-generic/errno.h)\ +//! C header: [`include/linux/errno.h`](srctree/include/linux/errno.h) + +use crate::{ + alloc::{layout::LayoutError, AllocError}, + fmt, + str::CStr, +}; + +use core::num::NonZeroI32; +use core::num::TryFromIntError; +use core::str::Utf8Error; + +/// Contains the C-compatible error codes. +#[rustfmt::skip] +pub mod code { + macro_rules! declare_err { + ($err:tt $(,)? $($doc:expr),+) => { + $( + #[doc = $doc] + )* + pub const $err: super::Error = + super::Error::try_from_errno(-(crate::bindings::$err as i32)) + .expect("Invalid errno in `declare_err!`"); + }; + } + + // From `include/uapi/asm-generic/errno-base.h`. + declare_err!(EPERM, "Operation not permitted."); + declare_err!(ENOENT, "No such file or directory."); + declare_err!(ESRCH, "No such process."); + declare_err!(EINTR, "Interrupted system call."); + declare_err!(EIO, "I/O error."); + declare_err!(ENXIO, "No such device or address."); + declare_err!(E2BIG, "Argument list too long."); + declare_err!(ENOEXEC, "Exec format error."); + declare_err!(EBADF, "Bad file number."); + declare_err!(ECHILD, "No child processes."); + declare_err!(EAGAIN, "Try again."); + declare_err!(ENOMEM, "Out of memory."); + declare_err!(EACCES, "Permission denied."); + declare_err!(EFAULT, "Bad address."); + declare_err!(ENOTBLK, "Block device required."); + declare_err!(EBUSY, "Device or resource busy."); + declare_err!(EEXIST, "File exists."); + declare_err!(EXDEV, "Cross-device link."); + declare_err!(ENODEV, "No such device."); + declare_err!(ENOTDIR, "Not a directory."); + declare_err!(EISDIR, "Is a directory."); + declare_err!(EINVAL, "Invalid argument."); + declare_err!(ENFILE, "File table overflow."); + declare_err!(EMFILE, "Too many open files."); + declare_err!(ENOTTY, "Not a typewriter."); + declare_err!(ETXTBSY, "Text file busy."); + declare_err!(EFBIG, "File too large."); + declare_err!(ENOSPC, "No space left on device."); + declare_err!(ESPIPE, "Illegal seek."); + declare_err!(EROFS, "Read-only file system."); + declare_err!(EMLINK, "Too many links."); + declare_err!(EPIPE, "Broken pipe."); + declare_err!(EDOM, "Math argument out of domain of func."); + declare_err!(ERANGE, "Math result not representable."); + + // From `include/uapi/asm-generic/errno.h`. + declare_err!(EDEADLK, "Resource deadlock would occur."); + declare_err!(ENAMETOOLONG, "File name too long."); + declare_err!(ENOLCK, "No record locks available."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EFSBADCRC, "Bad CRC detected."); + declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); + declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); + declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(EFSCORRUPTED, "Filesystem is corrupted."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(EFTYPE, "Wrong file type for the intended operation."); + + // From `include/linux/errno.h`. + declare_err!(ERESTARTSYS, "Restart the system call."); + declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); + declare_err!(ERESTARTNOHAND, "Restart if no handler."); + declare_err!(ENOIOCTLCMD, "No ioctl command."); + declare_err!(ERESTART_RESTARTBLOCK, "Restart by calling sys_restart_syscall."); + declare_err!(EPROBE_DEFER, "Driver requests probe retry."); + declare_err!(EOPENSTALE, "Open found a stale dentry."); + declare_err!(ENOPARAM, "Parameter not supported."); + declare_err!(EBADHANDLE, "Illegal NFS file handle."); + declare_err!(ENOTSYNC, "Update synchronization mismatch."); + declare_err!(EBADCOOKIE, "Cookie is stale."); + declare_err!(ENOTSUPP, "Operation is not supported."); + declare_err!(ETOOSMALL, "Buffer or request is too small."); + declare_err!(ESERVERFAULT, "An untranslatable error occurred."); + declare_err!(EBADTYPE, "Type not supported by server."); + declare_err!(EJUKEBOX, "Request initiated, but will not complete before timeout."); + declare_err!(EIOCBQUEUED, "iocb queued, will get completion event."); + declare_err!(ERECALLCONFLICT, "Conflict with recalled state."); + declare_err!(ENOGRACE, "NFS file lock reclaim refused."); +} + +/// Generic integer kernel error. +/// +/// The kernel defines a set of integer generic error codes based on C and +/// POSIX ones. These codes may have a more specific meaning in some contexts. +/// +/// # Invariants +/// +/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Error(NonZeroI32); + +impl Error { + /// Creates an [`Error`] from a kernel error code. + /// + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + /// + /// It is a bug to pass an out-of-range `errno`. [`code::EINVAL`] is returned in such a case. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Error::from_errno(-1), EPERM); + /// assert_eq!(Error::from_errno(-2), ENOENT); + /// ``` + /// + /// The following calls are considered a bug: + /// + /// ``` + /// assert_eq!(Error::from_errno(0), EINVAL); + /// assert_eq!(Error::from_errno(-1000000), EINVAL); + /// ``` + pub fn from_errno(errno: crate::ffi::c_int) -> Error { + if let Some(error) = Self::try_from_errno(errno) { + error + } else { + // TODO: Make it a `WARN_ONCE` once available. + crate::pr_warn!( + "attempted to create `Error` with out of range `errno`: {}\n", + errno + ); + code::EINVAL + } + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// Returns [`None`] if `errno` is out-of-range. + const fn try_from_errno(errno: crate::ffi::c_int) -> Option { + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + return None; + } + + // SAFETY: `errno` is checked above to be in a valid range. + Some(unsafe { Error::from_errno_unchecked(errno) }) + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// # Safety + /// + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + const unsafe fn from_errno_unchecked(errno: crate::ffi::c_int) -> Error { + // INVARIANT: The contract ensures the type invariant + // will hold. + // SAFETY: The caller guarantees `errno` is non-zero. + Error(unsafe { NonZeroI32::new_unchecked(errno) }) + } + + /// Returns the kernel error code. + pub fn to_errno(self) -> crate::ffi::c_int { + self.0.get() + } + + #[cfg(CONFIG_BLOCK)] + pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { + // SAFETY: `self.0` is a valid error due to its invariant. + unsafe { bindings::errno_to_blk_status(self.0.get()) } + } + + /// Returns the error encoded as a pointer. + pub fn to_ptr(self) -> *mut T { + // SAFETY: `self.0` is a valid error due to its invariant. + unsafe { bindings::ERR_PTR(self.0.get() as crate::ffi::c_long).cast() } + } + + /// Returns a string representing the error, if one exists. + #[cfg(not(testlib))] + pub fn name(&self) -> Option<&'static CStr> { + // SAFETY: Just an FFI call, there are no extra safety requirements. + let ptr = unsafe { bindings::errname(-self.0.get()) }; + if ptr.is_null() { + None + } else { + use crate::str::CStrExt as _; + + // SAFETY: The string returned by `errname` is static and `NUL`-terminated. + Some(unsafe { CStr::from_char_ptr(ptr) }) + } + } + + /// Returns a string representing the error, if one exists. + /// + /// When `testlib` is configured, this always returns `None` to avoid the dependency on a + /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still + /// run in userspace. + #[cfg(testlib)] + pub fn name(&self) -> Option<&'static CStr> { + None + } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.name() { + // Print out number if no name can be found. + None => f.debug_tuple("Error").field(&-self.0).finish(), + Some(name) => f + .debug_tuple( + // SAFETY: These strings are ASCII-only. + unsafe { core::str::from_utf8_unchecked(name.to_bytes()) }, + ) + .finish(), + } + } +} + +impl From for Error { + #[inline] + fn from(_: AllocError) -> Error { + code::ENOMEM + } +} + +impl From for Error { + #[inline] + fn from(_: TryFromIntError) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(_: Utf8Error) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(_: LayoutError) -> Error { + code::ENOMEM + } +} + +impl From for Error { + #[inline] + fn from(_: fmt::Error) -> Error { + code::EINVAL + } +} + +impl From for Error { + #[inline] + fn from(e: core::convert::Infallible) -> Error { + match e {} + } +} + +/// A [`Result`] with an [`Error`] error type. +/// +/// To be used as the return type for functions that may fail. +/// +/// # Error codes in C and Rust +/// +/// In C, it is common that functions indicate success or failure through +/// their return value; modifying or returning extra data through non-`const` +/// pointer parameters. In particular, in the kernel, functions that may fail +/// typically return an `int` that represents a generic error code. We model +/// those as [`Error`]. +/// +/// In Rust, it is idiomatic to model functions that may fail as returning +/// a [`Result`]. Since in the kernel many functions return an error code, +/// [`Result`] is a type alias for a [`core::result::Result`] that uses +/// [`Error`] as its error type. +/// +/// Note that even if a function does not return anything when it succeeds, +/// it should still be modeled as returning a [`Result`] rather than +/// just an [`Error`]. +/// +/// Calling a function that returns [`Result`] forces the caller to handle +/// the returned [`Result`]. +/// +/// This can be done "manually" by using [`match`]. Using [`match`] to decode +/// the [`Result`] is similar to C where all the return value decoding and the +/// error handling is done explicitly by writing handling code for each +/// error to cover. Using [`match`] the error and success handling can be +/// implemented in all detail as required. For example (inspired by +/// [`samples/rust/rust_minimal.rs`]): +/// +/// ``` +/// # #[allow(clippy::single_match)] +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// match numbers.push(72, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(108, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(200, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// An alternative to be more concise is the [`if let`] syntax: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// if let Err(e) = numbers.push(72, GFP_KERNEL) { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(108, GFP_KERNEL) { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(200, GFP_KERNEL) { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Instead of these verbose [`match`]/[`if let`], the [`?`] operator can +/// be used to handle the [`Result`]. Using the [`?`] operator is often +/// the best choice to handle [`Result`] in a non-verbose way as done in +/// [`samples/rust/rust_minimal.rs`]: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// numbers.push(72, GFP_KERNEL)?; +/// numbers.push(108, GFP_KERNEL)?; +/// numbers.push(200, GFP_KERNEL)?; +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Another possibility is to call [`unwrap()`](Result::unwrap) or +/// [`expect()`](Result::expect). However, use of these functions is +/// *heavily discouraged* in the kernel because they trigger a Rust +/// [`panic!`] if an error happens, which may destabilize the system or +/// entirely break it as a result -- just like the C [`BUG()`] macro. +/// Please see the documentation for the C macro [`BUG()`] for guidance +/// on when to use these functions. +/// +/// Alternatively, depending on the use case, using [`unwrap_or()`], +/// [`unwrap_or_else()`], [`unwrap_or_default()`] or [`unwrap_unchecked()`] +/// might be an option, as well. +/// +/// For even more details, please see the [Rust documentation]. +/// +/// [`match`]: https://doc.rust-lang.org/reference/expressions/match-expr.html +/// [`samples/rust/rust_minimal.rs`]: srctree/samples/rust/rust_minimal.rs +/// [`if let`]: https://doc.rust-lang.org/reference/expressions/if-expr.html#if-let-expressions +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator +/// [`unwrap()`]: Result::unwrap +/// [`expect()`]: Result::expect +/// [`BUG()`]: https://docs.kernel.org/process/deprecated.html#bug-and-bug-on +/// [`unwrap_or()`]: Result::unwrap_or +/// [`unwrap_or_else()`]: Result::unwrap_or_else +/// [`unwrap_or_default()`]: Result::unwrap_or_default +/// [`unwrap_unchecked()`]: Result::unwrap_unchecked +/// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html +pub type Result = core::result::Result; + +/// Converts an integer as returned by a C kernel function to a [`Result`]. +/// +/// If the integer is negative, an [`Err`] with an [`Error`] as given by [`Error::from_errno`] is +/// returned. This means the integer must be `>= -MAX_ERRNO`. +/// +/// Otherwise, it returns [`Ok`]. +/// +/// It is a bug to pass an out-of-range negative integer. `Err(EINVAL)` is returned in such a case. +/// +/// # Examples +/// +/// This function may be used to easily perform early returns with the [`?`] operator when working +/// with C APIs within Rust abstractions: +/// +/// ``` +/// # use kernel::error::to_result; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn f1() -> c_int { 0 } +/// # pub(super) unsafe fn f2() -> c_int { EINVAL.to_errno() } +/// # } +/// fn f() -> Result { +/// // SAFETY: ... +/// to_result(unsafe { bindings::f1() })?; +/// +/// // SAFETY: ... +/// to_result(unsafe { bindings::f2() })?; +/// +/// // ... +/// +/// Ok(()) +/// } +/// # assert_eq!(f(), Err(EINVAL)); +/// ``` +/// +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator +pub fn to_result(err: crate::ffi::c_int) -> Result { + if err < 0 { + Err(Error::from_errno(err)) + } else { + Ok(()) + } +} + +/// Transform a kernel "error pointer" to a normal pointer. +/// +/// Some kernel C API functions return an "error pointer" which optionally +/// embeds an `errno`. Callers are supposed to check the returned pointer +/// for errors. This function performs the check and converts the "error pointer" +/// to a normal pointer in an idiomatic fashion. +/// +/// Note that a `NULL` pointer is not considered an error pointer, and is returned +/// as-is, wrapped in [`Ok`]. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_err_ptr; +/// # use kernel::bindings; +/// fn devm_platform_ioremap_resource( +/// pdev: &mut PlatformDevice, +/// index: u32, +/// ) -> Result<*mut kernel::ffi::c_void> { +/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements +/// // on `index`. +/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) +/// } +/// ``` +/// +/// ``` +/// # use kernel::error::from_err_ptr; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn einval_err_ptr() -> *mut kernel::ffi::c_void { +/// # EINVAL.to_ptr() +/// # } +/// # pub(super) unsafe fn null_ptr() -> *mut kernel::ffi::c_void { +/// # core::ptr::null_mut() +/// # } +/// # pub(super) unsafe fn non_null_ptr() -> *mut kernel::ffi::c_void { +/// # 0x1234 as *mut kernel::ffi::c_void +/// # } +/// # } +/// // SAFETY: ... +/// let einval_err = from_err_ptr(unsafe { bindings::einval_err_ptr() }); +/// assert_eq!(einval_err, Err(EINVAL)); +/// +/// // SAFETY: ... +/// let null_ok = from_err_ptr(unsafe { bindings::null_ptr() }); +/// assert_eq!(null_ok, Ok(core::ptr::null_mut())); +/// +/// // SAFETY: ... +/// let non_null = from_err_ptr(unsafe { bindings::non_null_ptr() }).unwrap(); +/// assert_ne!(non_null, core::ptr::null_mut()); +/// ``` +pub fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { + // CAST: Casting a pointer to `*const crate::ffi::c_void` is always valid. + let const_ptr: *const crate::ffi::c_void = ptr.cast(); + // SAFETY: The FFI function does not deref the pointer. + if unsafe { bindings::IS_ERR(const_ptr) } { + // SAFETY: The FFI function does not deref the pointer. + let err = unsafe { bindings::PTR_ERR(const_ptr) }; + + #[allow(clippy::unnecessary_cast)] + // CAST: If `IS_ERR()` returns `true`, + // then `PTR_ERR()` is guaranteed to return a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`, + // which always fits in an `i16`, as per the invariant above. + // And an `i16` always fits in an `i32`. So casting `err` to + // an `i32` can never overflow, and is always valid. + // + // SAFETY: `IS_ERR()` ensures `err` is a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`. + return Err(unsafe { Error::from_errno_unchecked(err as crate::ffi::c_int) }); + } + Ok(ptr) +} + +/// Calls a closure returning a [`crate::error::Result`] and converts the result to +/// a C integer result. +/// +/// This is useful when calling Rust functions that return [`crate::error::Result`] +/// from inside `extern "C"` functions that need to return an integer error result. +/// +/// `T` should be convertible from an `i16` via `From`. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_result; +/// # use kernel::bindings; +/// unsafe extern "C" fn probe_callback( +/// pdev: *mut bindings::platform_device, +/// ) -> kernel::ffi::c_int { +/// from_result(|| { +/// let ptr = devm_alloc(pdev)?; +/// bindings::platform_set_drvdata(pdev, ptr); +/// Ok(0) +/// }) +/// } +/// ``` +pub fn from_result(f: F) -> T +where + T: From, + F: FnOnce() -> Result, +{ + match f() { + Ok(v) => v, + // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`, + // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above, + // therefore a negative `errno` always fits in an `i16` and will not overflow. + Err(e) => T::from(e.to_errno() as i16), + } +} + +/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait. +pub const VTABLE_DEFAULT_ERROR: &str = + "This function must not be called, see the #[vtable] documentation."; diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs new file mode 100644 index 000000000..cd4198fbb --- /dev/null +++ b/rust/kernel/faux.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the faux bus. +//! +//! This module provides bindings for working with faux devices in kernel modules. +//! +//! C header: [`include/linux/device/faux.h`](srctree/include/linux/device/faux.h) + +use crate::{ + bindings, + device, + prelude::*, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ptr::{ + null, + null_mut, + NonNull, // + }, +}; + +/// A faux device. +/// +/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a +/// real hardware device is not available or for testing. +/// +/// # Invariants +/// +/// The underlying `struct faux_device` is valid. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct faux_device`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self { + // SAFETY: `Device` is a transparent wrapper of `Opaque`. + unsafe { &*ptr.cast() } + } +} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct faux_device`. `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } +} + +// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`. +// The offset is guaranteed to point to a valid device field inside `faux::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev); +} + +/// The registration of a faux device. +/// +/// This type represents the registration of a [`struct faux_device`]. When an instance of this type +/// is dropped, its respective faux device will be unregistered from the system. +/// +/// # Invariants +/// +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. +/// +/// [`struct faux_device`]: srctree/include/linux/device/faux.h +pub struct Registration(NonNull); + +impl Registration { + /// Create and register a new faux device with the given name. + #[inline] + pub fn new(name: &CStr, parent: Option<&device::Device>) -> Result { + // SAFETY: + // - `name` is copied by this function into its own storage + // - `faux_ops` is safe to leave NULL according to the C API + // - `parent` can be either NULL or a pointer to a `struct device`, and `faux_device_create` + // will take a reference to `parent` using `device_add` - ensuring that it remains valid + // for the lifetime of the faux device. + let dev = unsafe { + bindings::faux_device_create( + name.as_char_ptr(), + parent.map_or(null_mut(), |p| p.as_raw()), + null(), + ) + }; + + // The above function will return either a valid device, or NULL on failure + // INVARIANT: The device will remain registered until faux_device_destroy() is called, which + // happens in our Drop implementation. + Ok(Self(NonNull::new(dev).ok_or(ENODEV)?)) + } + + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.as_ptr() + } +} + +impl AsRef> for Registration { + #[inline] + fn as_ref(&self) -> &Device { + // SAFETY: + // - The underlying `struct faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. + unsafe { Device::from_raw(self.as_raw()) } + } +} + +impl Drop for Registration { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.0` is a valid registered faux_device via our type invariants. + unsafe { bindings::faux_device_destroy(self.as_raw()) } + } +} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Send for Registration {} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Sync for Registration {} diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs new file mode 100644 index 000000000..a18f8b84f --- /dev/null +++ b/rust/kernel/firmware.rs @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Firmware abstraction +//! +//! C header: [`include/linux/firmware.h`](srctree/include/linux/firmware.h) + +use crate::{ + bindings, + device::Device, + error::to_result, + ffi, + prelude::*, + str::{CStr, CStrExt as _}, +}; +use core::ptr::NonNull; + +/// # Invariants +/// +/// One of the following: `bindings::request_firmware`, `bindings::firmware_request_nowarn`, +/// `bindings::firmware_request_platform`, `bindings::request_firmware_direct`. +struct FwFunc( + unsafe extern "C" fn( + *mut *const bindings::firmware, + *const ffi::c_char, + *mut bindings::device, + ) -> i32, +); + +impl FwFunc { + fn request() -> Self { + Self(bindings::request_firmware) + } + + fn request_nowarn() -> Self { + Self(bindings::firmware_request_nowarn) + } +} + +/// Abstraction around a C `struct firmware`. +/// +/// This is a simple abstraction around the C firmware API. Just like with the C API, firmware can +/// be requested. Once requested the abstraction provides direct access to the firmware buffer as +/// `&[u8]`. The firmware is released once [`Firmware`] is dropped. +/// +/// # Invariants +/// +/// The pointer is valid, and has ownership over the instance of `struct firmware`. +/// +/// The `Firmware`'s backing buffer is not modified. +/// +/// # Examples +/// +/// ```no_run +/// # use kernel::{device::Device, firmware::Firmware, sync::aref::ARef}; +/// # fn no_run(dev: ARef) -> Result<(), Error> { +/// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; +/// let blob = fw.data(); +/// +/// # Ok(()) +/// # } +/// ``` +pub struct Firmware(NonNull); + +impl Firmware { + fn request_internal(name: &CStr, dev: &Device, func: FwFunc) -> Result { + let mut fw: *mut bindings::firmware = core::ptr::null_mut(); + let pfw: *mut *mut bindings::firmware = &mut fw; + let pfw: *mut *const bindings::firmware = pfw.cast(); + + // SAFETY: `pfw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. + let ret = unsafe { func.0(pfw, name.as_char_ptr(), dev.as_raw()) }; + if ret != 0 { + return Err(Error::from_errno(ret)); + } + + // SAFETY: `func` not bailing out with a non-zero error code, guarantees that `fw` is a + // valid pointer to `bindings::firmware`. + Ok(Firmware(unsafe { NonNull::new_unchecked(fw) })) + } + + /// Send a firmware request and wait for it. See also `bindings::request_firmware`. + pub fn request(name: &CStr, dev: &Device) -> Result { + Self::request_internal(name, dev, FwFunc::request()) + } + + /// Send a request for an optional firmware module. See also + /// `bindings::firmware_request_nowarn`. + pub fn request_nowarn(name: &CStr, dev: &Device) -> Result { + Self::request_internal(name, dev, FwFunc::request_nowarn()) + } + + fn as_raw(&self) -> *mut bindings::firmware { + self.0.as_ptr() + } + + /// Returns the size of the requested firmware in bytes. + pub fn size(&self) -> usize { + // SAFETY: `self.as_raw()` is valid by the type invariant. + unsafe { (*self.as_raw()).size } + } + + /// Returns the requested firmware as `&[u8]`. + pub fn data(&self) -> &[u8] { + // SAFETY: `self.as_raw()` is valid by the type invariant. Additionally, + // `bindings::firmware` guarantees, if successfully requested, that + // `bindings::firmware::data` has a size of `bindings::firmware::size` bytes. + unsafe { core::slice::from_raw_parts((*self.as_raw()).data, self.size()) } + } +} + +impl Drop for Firmware { + fn drop(&mut self) { + // SAFETY: `self.as_raw()` is valid by the type invariant. + unsafe { bindings::release_firmware(self.as_raw()) }; + } +} + +/// Load firmware directly into the caller-provided `buf`. +/// +/// On success the firmware image has been copied into `buf`; the caller accesses the data +/// through `buf` itself. +/// +/// This is intentionally a stand-alone function rather than a `Firmware` constructor. For +/// the `into_buf` path, the firmware data lives in the caller's `buf`, not in a +/// kernel-owned buffer, so returning a `Firmware` would expose `Firmware::data()` as a +/// second handle aliasing `buf` (and `release_firmware()` does not free `buf` anyway). +pub fn request_into_buf(name: &CStr, dev: &Device, buf: &mut [u8]) -> Result { + // `as_mut_ptr()` on an empty slice returns a non-NULL pointer to + // memory which the loader does not own. Passing that pointer with `size == 0` + // makes the loader believe that it is buffer it allocated itself, so when + // `release_firmware()` is called, it will vfree the pointer and trigger a + // bug. Reject empty slices to avoid this situation. + if buf.is_empty() { + return Err(EINVAL); + } + + let mut fw: *const bindings::firmware = core::ptr::null(); + + // SAFETY: `&raw mut fw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. `buf` is a valid writable + // buffer of `buf.len()` bytes. + to_result(unsafe { + bindings::request_firmware_into_buf( + &raw mut fw, + name.as_char_ptr(), + dev.as_raw(), + buf.as_mut_ptr().cast(), + buf.len(), + ) + })?; + + // The firmware bytes are now in `buf`, which the caller owns, so we don't need + // the kernel to hang on to it any more. + // SAFETY: `fw` is a valid pointer returned by `request_firmware_into_buf`. + unsafe { bindings::release_firmware(fw) }; + + Ok(()) +} + +// SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, which is safe to be used from +// any thread. +unsafe impl Send for Firmware {} + +// SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, references to which are safe to +// be used from any thread. +unsafe impl Sync for Firmware {} + +/// Create firmware .modinfo entries. +/// +/// This macro is the counterpart of the C macro `MODULE_FIRMWARE()`, but instead of taking a +/// simple string literals, which is already covered by the `firmware` field of +/// [`crate::prelude::module!`], it allows the caller to pass a builder type, based on the +/// [`ModInfoBuilder`], which can create the firmware modinfo strings in a more flexible way. +/// +/// Drivers should extend the [`ModInfoBuilder`] with their own driver specific builder type. +/// +/// The `builder` argument must be a type which implements the following function. +/// +/// `const fn create(module_name: &'static CStr) -> ModInfoBuilder` +/// +/// `create` should pass the `module_name` to the [`ModInfoBuilder`] and, with the help of +/// it construct the corresponding firmware modinfo. +/// +/// Typically, such contracts would be enforced by a trait, however traits do not (yet) support +/// const functions. +/// +/// # Examples +/// +/// ``` +/// # mod module_firmware_test { +/// # use kernel::firmware; +/// # use kernel::prelude::*; +/// # +/// # struct MyModule; +/// # +/// # impl kernel::Module for MyModule { +/// # fn init(_module: &'static ThisModule) -> Result { +/// # Ok(Self) +/// # } +/// # } +/// # +/// # +/// struct Builder; +/// +/// impl Builder { +/// const DIR: &'static str = "vendor/chip/"; +/// const FILES: [&'static str; 3] = [ "foo", "bar", "baz" ]; +/// +/// const fn create(module_name: &'static kernel::str::CStr) -> firmware::ModInfoBuilder { +/// let mut builder = firmware::ModInfoBuilder::new(module_name); +/// +/// let mut i = 0; +/// while i < Self::FILES.len() { +/// builder = builder.new_entry() +/// .push(Self::DIR) +/// .push(Self::FILES[i]) +/// .push(".bin"); +/// +/// i += 1; +/// } +/// +/// builder +/// } +/// } +/// +/// module! { +/// type: MyModule, +/// name: "module_firmware_test", +/// authors: ["Rust for Linux"], +/// description: "module_firmware! test module", +/// license: "GPL", +/// } +/// +/// kernel::module_firmware!(Builder); +/// # } +/// ``` +#[macro_export] +macro_rules! module_firmware { + // The argument is the builder type without the const generic, since it's deferred from within + // this macro. Hence, we can neither use `expr` nor `ty`. + ($($builder:tt)*) => { + const _: () = { + const __MODULE_FIRMWARE_PREFIX: &'static $crate::str::CStr = if cfg!(MODULE) { + c"" + } else { + ::NAME + }; + + #[link_section = ".modinfo"] + #[used(compiler)] + static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX) + .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build(); + }; + }; +} + +/// Builder for firmware module info. +/// +/// [`ModInfoBuilder`] is a helper component to flexibly compose firmware paths strings for the +/// .modinfo section in const context. +/// +/// Therefore the [`ModInfoBuilder`] provides the methods [`ModInfoBuilder::new_entry`] and +/// [`ModInfoBuilder::push`], where the latter is used to push path components and the former to +/// mark the beginning of a new path string. +/// +/// [`ModInfoBuilder`] is meant to be used in combination with [`kernel::module_firmware!`]. +/// +/// The const generic `N` as well as the `module_name` parameter of [`ModInfoBuilder::new`] is an +/// internal implementation detail and supplied through the above macro. +pub struct ModInfoBuilder { + buf: [u8; N], + n: usize, + module_name: &'static CStr, +} + +impl ModInfoBuilder { + /// Create an empty builder instance. + pub const fn new(module_name: &'static CStr) -> Self { + Self { + buf: [0; N], + n: 0, + module_name, + } + } + + const fn push_internal(mut self, bytes: &[u8]) -> Self { + let mut j = 0; + + if N == 0 { + self.n += bytes.len(); + return self; + } + + while j < bytes.len() { + if self.n < N { + self.buf[self.n] = bytes[j]; + } + self.n += 1; + j += 1; + } + self + } + + /// Push an additional path component. + /// + /// Append path components to the [`ModInfoBuilder`] instance. Paths need to be separated + /// with [`ModInfoBuilder::new_entry`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::firmware::ModInfoBuilder; + /// + /// # const DIR: &str = "vendor/chip/"; + /// # const fn no_run(builder: ModInfoBuilder) { + /// let builder = builder.new_entry() + /// .push(DIR) + /// .push("foo.bin") + /// .new_entry() + /// .push(DIR) + /// .push("bar.bin"); + /// # } + /// ``` + pub const fn push(self, s: &str) -> Self { + // Check whether there has been an initial call to `next_entry()`. + if N != 0 && self.n == 0 { + crate::build_error!("Must call next_entry() before push()."); + } + + self.push_internal(s.as_bytes()) + } + + const fn push_module_name(self) -> Self { + let mut this = self; + let module_name = this.module_name; + + if !this.module_name.is_empty() { + this = this.push_internal(module_name.to_bytes_with_nul()); + + if N != 0 { + // Re-use the space taken by the NULL terminator and swap it with the '.' separator. + this.buf[this.n - 1] = b'.'; + } + } + + this + } + + /// Prepare the [`ModInfoBuilder`] for the next entry. + /// + /// This method acts as a separator between module firmware path entries. + /// + /// Must be called before constructing a new entry with subsequent calls to + /// [`ModInfoBuilder::push`]. + /// + /// See [`ModInfoBuilder::push`] for an example. + pub const fn new_entry(self) -> Self { + self.push_internal(b"\0") + .push_module_name() + .push_internal(b"firmware=") + } + + /// Build the byte array. + pub const fn build(self) -> [u8; N] { + // Add the final NULL terminator. + let this = self.push_internal(b"\0"); + + if this.n == N { + this.buf + } else { + crate::build_error!("Length mismatch."); + } + } +} + +impl ModInfoBuilder<0> { + /// Return the length of the byte array to build. + pub const fn build_length(self) -> usize { + // Compensate for the NULL terminator added by `build`. + self.n + 1 + } +} diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs new file mode 100644 index 000000000..29582b053 --- /dev/null +++ b/rust/kernel/fmt.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Formatting utilities. +//! +//! This module is intended to be used in place of `core::fmt` in kernel code. + +use kernel::prelude::*; + +pub use core::fmt::{ + Arguments, + Debug, + Error, + Formatter, + Result, + Write, // +}; + +/// Internal adapter used to route and allow implementations of formatting traits for foreign types. +/// +/// It is inserted automatically by the [`fmt!`] macro and is not meant to be used directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +#[doc(hidden)] +pub struct Adapter(pub T); + +macro_rules! impl_fmt_adapter_forward { + ($($trait:ident),* $(,)?) => { + $( + impl $trait for Adapter { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + $trait::fmt(t, f) + } + } + )* + }; +} + +use core::fmt::{ + Binary, + LowerExp, + LowerHex, + Octal, + UpperExp, + UpperHex, // +}; +use core::ptr::NonNull; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Pointer`] that allows implementing pointer formatting for foreign types. +/// +/// Together with the [`Adapter`] type and [`fmt!`] macro, it enables raw pointer formatting to be +/// intercepted and routed to [`HashedPtr`] (kernel's `%p` hashed format), preventing kernel address +/// leaks. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Pointer { + /// Same as [`core::fmt::Pointer::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +/// A wrapper for pointers that formats them using kernel's `%p` format specifier. +/// +/// By default, `%p` prints a hashed representation of the pointer address to prevent kernel address +/// leaks. When the `no_hash_pointers` kernel command-line parameter is enabled, the real address is +/// printed instead (for debugging purposes). +pub struct HashedPtr(pub *const T); + +impl Pointer for HashedPtr { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + use crate::str::CStrExt as _; + + let mut buf = [0u8; 32]; + + // Use `%#0*p` for the `0x` prefix and zero-padding; `+2` compensates for + // the prefix counting toward the field width. + let default_width = (2 * size_of::() + 2) as c_int; + let width = match (f.sign_aware_zero_pad(), f.width()) { + (true, Some(w)) if w > 0 => w.min(buf.len() - 1) as c_int, + _ => default_width, + }; + + // SAFETY: `buf` is a valid, writable 32-byte buffer, sufficient for + // all architectures (max 19 bytes for 64-bit under the default width). + // The format string is null-terminated; `width` (c_int) and pointer + // match the `%*` and `%p` specifiers. + let len = unsafe { + crate::bindings::scnprintf( + buf.as_mut_ptr().cast(), + buf.len(), + c"%#0*p".as_char_ptr(), + width, + self.0.cast::(), + ) + }; + + // SAFETY: `%#0*p` produces only ASCII, which is valid UTF-8. + let s = unsafe { core::str::from_utf8_unchecked(&buf[..len as usize]) }; + + if f.sign_aware_zero_pad() { + // `scnprintf` already applied the width and zero-padding via `%#0*p`. + f.write_str(s) + } else { + f.pad(s) + } + } +} + +// Raw pointers are formatted via `HashedPtr` (kernel `%p`: hashed by default, plain with +// `no_hash_pointers`). +impl Pointer for *const T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for *mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for &T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl Pointer for &mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(core::ptr::from_ref(*self)), f) + } +} + +impl Pointer for NonNull { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(self.as_ptr()), f) + } +} + +// `Adapter<&T>` bridges our `Pointer` trait to `core::fmt::Pointer` +impl core::fmt::Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} + +/// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. +/// +/// Types should implement this trait rather than [`core::fmt::Display`]. Together with the +/// [`Adapter`] type and [`fmt!`] macro, it allows for formatting foreign types (e.g. types from +/// core) which do not implement [`core::fmt::Display`] directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Display { + /// Same as [`core::fmt::Display::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +impl Display for &T { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Display::fmt(*self, f) + } +} + +impl core::fmt::Display for Adapter<&T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + Display::fmt(t, f) + } +} + +macro_rules! impl_display_forward { + ($( + $( { $($generics:tt)* } )? $ty:ty $( { where $($where:tt)* } )? + ),* $(,)?) => { + $( + impl$($($generics)*)? Display for $ty $(where $($where)*)? { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + core::fmt::Display::fmt(self, f) + } + } + )* + }; +} + +impl_display_forward!( + bool, + char, + core::panic::PanicInfo<'_>, + Arguments<'_>, + i128, + i16, + i32, + i64, + i8, + isize, + str, + u128, + u16, + u32, + u64, + u8, + usize, + {} crate::sync::Arc {where crate::sync::Arc: core::fmt::Display}, + {} crate::sync::UniqueArc {where crate::sync::UniqueArc: core::fmt::Display}, +); + +#[macros::kunit_tests(rust_kernel_fmt)] +mod tests { + use crate::{ + bindings, + prelude::fmt, + str::CString, // + }; + + #[cfg(CONFIG_64BIT)] + mod expected { + pub(super) const PTR_VALUE: usize = 0xffffffffdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(____ptrval____)"; + pub(super) const HASHED_PREFIX: &str = "0x00000000"; + pub(super) const RAW_POINTER: &str = "0xffffffffdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xffffffffdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x000000ffffffffdeadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000ffffffffdeadbeef"; + } + + #[cfg(not(CONFIG_64BIT))] + mod expected { + pub(super) const PTR_VALUE: usize = 0xdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(ptrval)"; + pub(super) const HASHED_PREFIX: &str = "0x"; + pub(super) const RAW_POINTER: &str = "0xdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x00000000000000deadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000000000000deadbeef"; + } + + #[test] + fn test_ptr_formatting() -> core::result::Result<(), crate::error::Error> { + let ptr: *const u8 = core::ptr::without_provenance(expected::PTR_VALUE); + + // SAFETY: `no_hash_pointers` is a global variable that is never concurrently modified — + // KUnit tests may run at boot (before `mark_readonly()`) or manually afterwards (when the + // variable is read-only). Reading is always safe. + let no_hash = unsafe { bindings::no_hash_pointers }; + + if no_hash { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::PADDED_RIGHT); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::ZERO_PADDED); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::CLAMPED); + } else { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + let formatted = cstr.to_str()?; + // If the RNG is not yet ready, `%p` falls back to a placeholder. + if formatted == expected::PTR_VAL_NO_CRNG { + return Ok(()); + } + assert!(formatted.starts_with(expected::HASHED_PREFIX)); + assert_ne!(formatted, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_PADDED_RIGHT_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_ZERO_PADDED_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + let output = cstr.to_str()?; + assert!(output.starts_with("0x")); + assert!(!output[2..].chars().all(|c| c == '0')); + } + + Ok(()) + } +} diff --git a/rust/kernel/fs.rs b/rust/kernel/fs.rs new file mode 100644 index 000000000..6ba6bdf14 --- /dev/null +++ b/rust/kernel/fs.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel file systems. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) + +pub mod file; +pub use self::file::{File, LocalFile}; + +mod kiocb; +pub use self::kiocb::Kiocb; diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs new file mode 100644 index 000000000..23ee689bd --- /dev/null +++ b/rust/kernel/fs/file.rs @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Files and file descriptors. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) and +//! [`include/linux/file.h`](srctree/include/linux/file.h) + +use crate::{ + bindings, + cred::Credential, + error::{code::*, to_result, Error, Result}, + fmt, + sync::aref::{ARef, AlwaysRefCounted}, + types::{NotThreadSafe, Opaque}, +}; +use core::ptr; + +/// Primitive type representing the offset within a [`File`]. +/// +/// Type alias for `bindings::loff_t`. +pub type Offset = bindings::loff_t; + +/// Flags associated with a [`File`]. +pub mod flags { + /// File is opened in append mode. + pub const O_APPEND: u32 = bindings::O_APPEND; + + /// Signal-driven I/O is enabled. + pub const O_ASYNC: u32 = bindings::FASYNC; + + /// Close-on-exec flag is set. + pub const O_CLOEXEC: u32 = bindings::O_CLOEXEC; + + /// File was created if it didn't already exist. + pub const O_CREAT: u32 = bindings::O_CREAT; + + /// Direct I/O is enabled for this file. + pub const O_DIRECT: u32 = bindings::O_DIRECT; + + /// File must be a directory. + pub const O_DIRECTORY: u32 = bindings::O_DIRECTORY; + + /// Like [`O_SYNC`] except metadata is not synced. + pub const O_DSYNC: u32 = bindings::O_DSYNC; + + /// Ensure that this file is created with the `open(2)` call. + pub const O_EXCL: u32 = bindings::O_EXCL; + + /// Large file size enabled (`off64_t` over `off_t`). + pub const O_LARGEFILE: u32 = bindings::O_LARGEFILE; + + /// Do not update the file last access time. + pub const O_NOATIME: u32 = bindings::O_NOATIME; + + /// File should not be used as process's controlling terminal. + pub const O_NOCTTY: u32 = bindings::O_NOCTTY; + + /// If basename of path is a symbolic link, fail open. + pub const O_NOFOLLOW: u32 = bindings::O_NOFOLLOW; + + /// File is using nonblocking I/O. + pub const O_NONBLOCK: u32 = bindings::O_NONBLOCK; + + /// File is using nonblocking I/O. + /// + /// This is effectively the same flag as [`O_NONBLOCK`] on all architectures + /// except SPARC64. + pub const O_NDELAY: u32 = bindings::O_NDELAY; + + /// Used to obtain a path file descriptor. + pub const O_PATH: u32 = bindings::O_PATH; + + /// Write operations on this file will flush data and metadata. + pub const O_SYNC: u32 = bindings::O_SYNC; + + /// This file is an unnamed temporary regular file. + pub const O_TMPFILE: u32 = bindings::O_TMPFILE; + + /// File should be truncated to length 0. + pub const O_TRUNC: u32 = bindings::O_TRUNC; + + /// Bitmask for access mode flags. + /// + /// # Examples + /// + /// ``` + /// use kernel::fs::file; + /// # fn do_something() {} + /// # let flags = 0; + /// if (flags & file::flags::O_ACCMODE) == file::flags::O_RDONLY { + /// do_something(); + /// } + /// ``` + pub const O_ACCMODE: u32 = bindings::O_ACCMODE; + + /// File is read only. + pub const O_RDONLY: u32 = bindings::O_RDONLY; + + /// File is write only. + pub const O_WRONLY: u32 = bindings::O_WRONLY; + + /// File can be both read and written. + pub const O_RDWR: u32 = bindings::O_RDWR; +} + +/// Wraps the kernel's `struct file`. Thread safe. +/// +/// This represents an open file rather than a file on a filesystem. Processes generally reference +/// open files using file descriptors. However, file descriptors are not the same as files. A file +/// descriptor is just an integer that corresponds to a file, and a single file may be referenced +/// by multiple file descriptors. +/// +/// # Refcounting +/// +/// Instances of this type are reference-counted. The reference count is incremented by the +/// `fget`/`get_file` functions and decremented by `fput`. The Rust type `ARef` represents a +/// pointer that owns a reference count on the file. +/// +/// Whenever a process opens a file descriptor (fd), it stores a pointer to the file in its fd +/// table (`struct files_struct`). This pointer owns a reference count to the file, ensuring the +/// file isn't prematurely deleted while the file descriptor is open. In Rust terminology, the +/// pointers in `struct files_struct` are `ARef` pointers. +/// +/// ## Light refcounts +/// +/// Whenever a process has an fd to a file, it may use something called a "light refcount" as a +/// performance optimization. Light refcounts are acquired by calling `fdget` and released with +/// `fdput`. The idea behind light refcounts is that if the fd is not closed between the calls to +/// `fdget` and `fdput`, then the refcount cannot hit zero during that time, as the `struct +/// files_struct` holds a reference until the fd is closed. This means that it's safe to access the +/// file even if `fdget` does not increment the refcount. +/// +/// The requirement that the fd is not closed during a light refcount applies globally across all +/// threads - not just on the thread using the light refcount. For this reason, light refcounts are +/// only used when the `struct files_struct` is not shared with other threads, since this ensures +/// that other unrelated threads cannot suddenly start using the fd and close it. Therefore, +/// calling `fdget` on a shared `struct files_struct` creates a normal refcount instead of a light +/// refcount. +/// +/// Light reference counts must be released with `fdput` before the system call returns to +/// userspace. This means that if you wait until the current system call returns to userspace, then +/// all light refcounts that existed at the time have gone away. +/// +/// ### The file position +/// +/// Each `struct file` has a position integer, which is protected by the `f_pos_lock` mutex. +/// However, if the `struct file` is not shared, then the kernel may avoid taking the lock as a +/// performance optimization. +/// +/// The condition for avoiding the `f_pos_lock` mutex is different from the condition for using +/// `fdget`. With `fdget`, you may avoid incrementing the refcount as long as the current fd table +/// is not shared; it is okay if there are other fd tables that also reference the same `struct +/// file`. However, `fdget_pos` can only avoid taking the `f_pos_lock` if the entire `struct file` +/// is not shared, as different processes with an fd to the same `struct file` share the same +/// position. +/// +/// To represent files that are not thread safe due to this optimization, the [`LocalFile`] type is +/// used. +/// +/// ## Rust references +/// +/// The reference type `&File` is similar to light refcounts: +/// +/// * `&File` references don't own a reference count. They can only exist as long as the reference +/// count stays positive, and can only be created when there is some mechanism in place to ensure +/// this. +/// +/// * The Rust borrow-checker normally ensures this by enforcing that the `ARef` from which +/// a `&File` is created outlives the `&File`. +/// +/// * Using the unsafe [`File::from_raw_file`] means that it is up to the caller to ensure that the +/// `&File` only exists while the reference count is positive. +/// +/// * You can think of `fdget` as using an fd to look up an `ARef` in the `struct +/// files_struct` and create an `&File` from it. The "fd cannot be closed" rule is like the Rust +/// rule "the `ARef` must outlive the `&File`". +/// +/// # Invariants +/// +/// * All instances of this type are refcounted using the `f_count` field. +/// * There must not be any active calls to `fdget_pos` on this file that did not take the +/// `f_pos_lock` mutex. +#[repr(transparent)] +pub struct File { + inner: Opaque, +} + +// SAFETY: This file is known to not have any active `fdget_pos` calls that did not take the +// `f_pos_lock` mutex, so it is safe to transfer it between threads. +unsafe impl Send for File {} + +// SAFETY: This file is known to not have any active `fdget_pos` calls that did not take the +// `f_pos_lock` mutex, so it is safe to access its methods from several threads in parallel. +unsafe impl Sync for File {} + +// SAFETY: The type invariants guarantee that `File` is always ref-counted. This implementation +// makes `ARef` own a normal refcount. +unsafe impl AlwaysRefCounted for File { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_file(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: To call this method, the caller passes us ownership of a normal refcount, so we + // may drop it. The cast is okay since `File` has the same representation as `struct file`. + unsafe { bindings::fput(obj.cast().as_ptr()) } + } +} + +/// Wraps the kernel's `struct file`. Not thread safe. +/// +/// This type represents a file that is not known to be safe to transfer across thread boundaries. +/// To obtain a thread-safe [`File`], use the [`assume_no_fdget_pos`] conversion. +/// +/// See the documentation for [`File`] for more information. +/// +/// # Invariants +/// +/// * All instances of this type are refcounted using the `f_count` field. +/// * If there is an active call to `fdget_pos` that did not take the `f_pos_lock` mutex, then it +/// must be on the same thread as this file. +/// +/// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos +#[repr(transparent)] +pub struct LocalFile { + inner: Opaque, +} + +// SAFETY: The type invariants guarantee that `LocalFile` is always ref-counted. This implementation +// makes `ARef` own a normal refcount. +unsafe impl AlwaysRefCounted for LocalFile { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_file(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: To call this method, the caller passes us ownership of a normal refcount, so we + // may drop it. The cast is okay since `LocalFile` has the same representation as + // `struct file`. + unsafe { bindings::fput(obj.cast().as_ptr()) } + } +} + +impl LocalFile { + /// Constructs a new `struct file` wrapper from a file descriptor. + /// + /// The file descriptor belongs to the current process, and there might be active local calls + /// to `fdget_pos` on the same file. + /// + /// To obtain an `ARef`, use the [`assume_no_fdget_pos`] function to convert. + /// + /// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos + #[inline] + pub fn fget(fd: u32) -> Result, BadFdError> { + // SAFETY: FFI call, there are no requirements on `fd`. + let ptr = ptr::NonNull::new(unsafe { bindings::fget(fd) }).ok_or(BadFdError)?; + + // SAFETY: `bindings::fget` created a refcount, and we pass ownership of it to the `ARef`. + // + // INVARIANT: This file is in the fd table on this thread, so either all `fdget_pos` calls + // are on this thread, or the file is shared, in which case `fdget_pos` calls took the + // `f_pos_lock` mutex. + Ok(unsafe { ARef::from_raw(ptr.cast()) }) + } + + /// Creates a reference to a [`LocalFile`] from a valid pointer. + /// + /// # Safety + /// + /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is + /// positive for the duration of `'a`. + /// * The caller must ensure that if there is an active call to `fdget_pos` that did not take + /// the `f_pos_lock` mutex, then that call is on the current thread. + #[inline] + pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a LocalFile { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of `'a`. The cast is okay because `LocalFile` is `repr(transparent)`. + // + // INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls. + unsafe { &*ptr.cast() } + } + + /// Assume that there are no active `fdget_pos` calls that prevent us from sharing this file. + /// + /// This makes it safe to transfer this file to other threads. No checks are performed, and + /// using it incorrectly may lead to a data race on the file position if the file is shared + /// with another thread. + /// + /// This method is intended to be used together with [`LocalFile::fget`] when the caller knows + /// statically that there are no `fdget_pos` calls on the current thread. For example, you + /// might use it when calling `fget` from an ioctl, since ioctls usually do not touch the file + /// position. + /// + /// # Safety + /// + /// There must not be any active `fdget_pos` calls on the current thread. + #[inline] + pub unsafe fn assume_no_fdget_pos(me: ARef) -> ARef { + // INVARIANT: There are no `fdget_pos` calls on the current thread, and by the type + // invariants, if there is a `fdget_pos` call on another thread, then it took the + // `f_pos_lock` mutex. + // + // SAFETY: `LocalFile` and `File` have the same layout. + unsafe { ARef::from_raw(ARef::into_raw(me).cast()) } + } + + /// Returns a raw pointer to the inner C struct. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::file { + self.inner.get() + } + + /// Returns the credentials of the task that originally opened the file. + pub fn cred(&self) -> &Credential { + // SAFETY: It's okay to read the `f_cred` field without synchronization because `f_cred` is + // never changed after initialization of the file. + let ptr = unsafe { (*self.as_ptr()).f_cred }; + + // SAFETY: The signature of this function ensures that the caller will only access the + // returned credential while the file is still valid, and the C side ensures that the + // credential stays valid at least as long as the file. + unsafe { Credential::from_ptr(ptr) } + } + + /// Returns the flags associated with the file. + /// + /// The flags are a combination of the constants in [`flags`]. + #[inline] + pub fn flags(&self) -> u32 { + // This `read_volatile` is intended to correspond to a READ_ONCE call. + // + // SAFETY: The file is valid because the shared reference guarantees a nonzero refcount. + // + // FIXME(read_once): Replace with `read_once` when available on the Rust side. + unsafe { core::ptr::addr_of!((*self.as_ptr()).f_flags).read_volatile() } + } +} + +impl File { + /// Creates a reference to a [`File`] from a valid pointer. + /// + /// # Safety + /// + /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is + /// positive for the duration of `'a`. + /// * The caller must ensure that if there are active `fdget_pos` calls on this file, then they + /// took the `f_pos_lock` mutex. + #[inline] + pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a File { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of `'a`. The cast is okay because `File` is `repr(transparent)`. + // + // INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls. + unsafe { &*ptr.cast() } + } +} + +// Make LocalFile methods available on File. +impl core::ops::Deref for File { + type Target = LocalFile; + #[inline] + fn deref(&self) -> &LocalFile { + // SAFETY: The caller provides a `&File`, and since it is a reference, it must point at a + // valid file for the desired duration. + // + // By the type invariants, there are no `fdget_pos` calls that did not take the + // `f_pos_lock` mutex. + unsafe { LocalFile::from_raw_file(core::ptr::from_ref(self).cast()) } + } +} + +/// A file descriptor reservation. +/// +/// This allows the creation of a file descriptor in two steps: first, we reserve a slot for it, +/// then we commit or drop the reservation. The first step may fail (e.g., the current process ran +/// out of available slots), but commit and drop never fail (and are mutually exclusive). +/// +/// Dropping the reservation happens in the destructor of this type. +/// +/// # Invariants +/// +/// The fd stored in this struct must correspond to a reserved file descriptor of the current task. +pub struct FileDescriptorReservation { + fd: u32, + /// Prevent values of this type from being moved to a different task. + /// + /// The `fd_install` and `put_unused_fd` functions assume that the value of `current` is + /// unchanged since the call to `get_unused_fd_flags`. By adding this marker to this type, we + /// prevent it from being moved across task boundaries, which ensures that `current` does not + /// change while this value exists. + _not_send: NotThreadSafe, +} + +impl FileDescriptorReservation { + /// Creates a new file descriptor reservation. + #[inline] + pub fn get_unused_fd_flags(flags: u32) -> Result { + // SAFETY: FFI call, there are no safety requirements on `flags`. + let fd: i32 = unsafe { bindings::get_unused_fd_flags(flags) }; + to_result(fd)?; + + Ok(Self { + fd: fd as u32, + _not_send: NotThreadSafe, + }) + } + + /// Returns the file descriptor number that was reserved. + #[inline] + pub fn reserved_fd(&self) -> u32 { + self.fd + } + + /// Commits the reservation. + /// + /// The previously reserved file descriptor is bound to `file`. This method consumes the + /// [`FileDescriptorReservation`], so it will not be usable after this call. + #[inline] + pub fn fd_install(self, file: ARef) { + // SAFETY: `self.fd` was previously returned by `get_unused_fd_flags`. We have not yet used + // the fd, so it is still valid, and `current` still refers to the same task, as this type + // cannot be moved across task boundaries. + // + // Furthermore, the file pointer is guaranteed to own a refcount by its type invariants, + // and we take ownership of that refcount by not running the destructor below. + // Additionally, the file is known to not have any non-shared `fdget_pos` calls, so even if + // this process starts using the file position, this will not result in a data race on the + // file position. + unsafe { bindings::fd_install(self.fd, file.as_ptr()) }; + + // `fd_install` consumes both the file descriptor and the file reference, so we cannot run + // the destructors. + core::mem::forget(self); + core::mem::forget(file); + } +} + +impl Drop for FileDescriptorReservation { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants of this type, `self.fd` was previously returned by + // `get_unused_fd_flags`. We have not yet used the fd, so it is still valid, and `current` + // still refers to the same task, as this type cannot be moved across task boundaries. + unsafe { bindings::put_unused_fd(self.fd) }; + } +} + +/// Represents the [`EBADF`] error code. +/// +/// Used for methods that can only fail with [`EBADF`]. +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct BadFdError; + +impl From for Error { + #[inline] + fn from(_: BadFdError) -> Error { + EBADF + } +} + +impl fmt::Debug for BadFdError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("EBADF") + } +} diff --git a/rust/kernel/fs/kiocb.rs b/rust/kernel/fs/kiocb.rs new file mode 100644 index 000000000..84c936cd6 --- /dev/null +++ b/rust/kernel/fs/kiocb.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Kernel IO callbacks. +//! +//! C headers: [`include/linux/fs.h`](srctree/include/linux/fs.h) + +use core::marker::PhantomData; +use core::ptr::NonNull; +use kernel::types::ForeignOwnable; + +/// Wrapper for the kernel's `struct kiocb`. +/// +/// Currently this abstractions is incomplete and is essentially just a tuple containing a +/// reference to a file and a file position. +/// +/// The type `T` represents the filesystem or driver specific data associated with the file. +/// +/// # Invariants +/// +/// `inner` points at a valid `struct kiocb` whose file has the type `T` as its private data. +pub struct Kiocb<'a, T> { + inner: NonNull, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T: ForeignOwnable> Kiocb<'a, T> { + /// Create a `Kiocb` from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference a valid `struct kiocb` for the duration of `'a`. The private + /// data of the file must be `T`. + pub unsafe fn from_raw(kiocb: *mut bindings::kiocb) -> Self { + Self { + // SAFETY: If a pointer is valid it is not null. + inner: unsafe { NonNull::new_unchecked(kiocb) }, + _phantom: PhantomData, + } + } + + /// Access the underlying `struct kiocb` directly. + pub fn as_raw(&self) -> *mut bindings::kiocb { + self.inner.as_ptr() + } + + /// Get the filesystem or driver specific data associated with the file. + pub fn file(&self) -> ::Borrowed<'a> { + // SAFETY: We have shared access to this kiocb and hence the underlying file, so we can + // read the file's private data. + let private = unsafe { (*(*self.as_raw()).ki_filp).private_data }; + // SAFETY: The kiocb has shared access to the private data. + unsafe { ::borrow(private) } + } + + /// Gets the current value of `ki_pos`. + pub fn ki_pos(&self) -> i64 { + // SAFETY: We have shared access to the kiocb, so we can read its `ki_pos` field. + unsafe { (*self.as_raw()).ki_pos } + } + + /// Gets a mutable reference to the `ki_pos` field. + pub fn ki_pos_mut(&mut self) -> &mut i64 { + // SAFETY: We have exclusive access to the kiocb, so we can write to `ki_pos`. + unsafe { &mut (*self.as_raw()).ki_pos } + } +} diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs new file mode 100644 index 000000000..f29244fb0 --- /dev/null +++ b/rust/kernel/fwctl.rs @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the fwctl subsystem. +//! +//! C header: `include/linux/fwctl.h` + +use crate::{ + bindings, + container_of, + device, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + ptr::NonNull, + slice, // +}; + +/// Returns a kmalloc-compatible allocation size for `T`. +const fn kmalloc_aligned_size() -> usize { + Layout::new::().pad_to_align().size() +} + +/// Represents a fwctl device type. +/// +/// Corresponds to the C `enum fwctl_device_type`. All non-error UAPI values are represented so +/// Rust drivers can select a device type without passing an untyped integer, while +/// `FWCTL_DEVICE_TYPE_ERROR` remains unrepresentable. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum DeviceType { + /// Mellanox ConnectX (mlx5) device. + Mlx5 = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_MLX5, + /// CXL (Compute Express Link) device. + Cxl = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_CXL, + /// AMD/Pensando PDS device. + Pds = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_PDS, + /// Broadcom NetXtreme (bnxt) device. + Bnxt = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_BNXT, +} + +/// Scope of access for an RPC request. +/// +/// Corresponds to the C `enum fwctl_rpc_scope`. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RpcScope { + /// Read/write access to device configuration. + Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION, + /// Read-only access to debug information. + DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY, + /// Write access to lockdown-compatible debug information. + DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE, + /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`). + DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL, +} + +impl TryFrom for RpcScope { + type Error = Error; + + #[inline] + fn try_from(value: u32) -> Result { + match value { + v if v == Self::Configuration as u32 => Ok(Self::Configuration), + v if v == Self::DebugReadOnly as u32 => Ok(Self::DebugReadOnly), + v if v == Self::DebugWrite as u32 => Ok(Self::DebugWrite), + v if v == Self::DebugWriteFull as u32 => Ok(Self::DebugWriteFull), + _ => Err(EINVAL), + } + } +} + +/// Response from a [`Operations::fw_rpc`] call. +pub enum FwRpcResponse { + /// Reuse the input buffer as the output, with the given output length. + /// + /// The callback returns `EINVAL` if the output length exceeds the input buffer length. + InPlace(usize), + /// Return a newly allocated buffer as the output. + NewBuffer(KVVec), +} + +/// Trait implemented by each Rust driver that integrates with the fwctl subsystem. +/// +/// The implementing type **is** the per-FD user context: one instance is +/// created for each `open()` call and dropped when the FD is closed. +/// +/// Each implementation corresponds to a specific device type and provides the +/// vtable used by the core `fwctl` layer to manage per-FD user contexts and +/// handle RPC requests. +pub trait Operations: Sized + Send + Sync + 'static { + /// Data owned by the [`Registration`] and accessible during callbacks. + /// + /// The lifetime `'a` is tied to the [`Registration`] scope (which lives within the parent bus + /// device binding scope). Drivers use it to store references to resources bound to this scope, + /// such as PCI BARs or typed bus device references. + type RegistrationData<'a>: Send + Sync + 'a + where + Self: 'a; + + /// fwctl device type identifier. + const DEVICE_TYPE: DeviceType; + + /// Called when a new user context is opened. + /// + /// Returns a [`PinInit`] initializer for `Self`. The instance is dropped + /// automatically when the FD is closed (after [`close`](Self::close)). + fn open<'a>( + device: &Device, + reg_data: &Self::RegistrationData<'a>, + ) -> impl PinInit; + + /// Called when the user context is closed. + /// + /// The driver may perform additional cleanup here that requires access + /// to the owning [`Device`]. `Self` is dropped automatically after this + /// returns. + fn close<'a>( + _this: Pin<&mut Self>, + _device: &Device, + _reg_data: &Self::RegistrationData<'a>, + ) { + } + + /// Return device information to userspace. + /// + /// The default implementation returns no device-specific data. + fn info<'a>( + _this: Pin<&Self>, + _device: &Device, + _reg_data: &Self::RegistrationData<'a>, + ) -> Result, Error> { + Ok(KVec::new()) + } + + /// Handle a userspace RPC request. + /// + /// `max_output_len` is the size of the userspace output buffer. A driver may return a larger + /// response to report the required size; the fwctl core copies only the bytes that fit and + /// reports the full response length to userspace. + fn fw_rpc<'a>( + this: Pin<&Self>, + device: &Device, + reg_data: &Self::RegistrationData<'a>, + scope: RpcScope, + rpc_buf: &mut [u8], + max_output_len: usize, + ) -> Result; +} + +/// A fwctl device. +/// +/// `#[repr(C)]` with the `fwctl_device` at offset 0, matching the C `fwctl_alloc_device()` layout +/// convention. Contains a pointer to the [`Registration`]'s data, set at registration time and +/// cleared on unregistration. +/// +/// # Invariants +/// +/// - `dev` is embedded at offset 0 and is initialised by fwctl. +/// - The fwctl refcount owns the allocation lifetime. +/// - `registration_data` is either [`NonNull::dangling()`] (before registration / after +/// unregistration) or points to valid data owned by the [`Registration`]. +#[repr(C)] +pub struct Device { + dev: Opaque, + registration_data: UnsafeCell>>, +} + +impl Device { + /// Allocate a new fwctl device. + /// + /// Returns an [`ARef`] that can be passed to [`Registration::new()`] + /// to make the device visible to userspace. + pub fn new(parent: &device::Device) -> Result> { + const_assert!( + core::mem::offset_of!(Self, dev) == 0, + "struct fwctl_device must be at offset 0" + ); + + let size = kmalloc_aligned_size::(); + let ops = core::ptr::from_ref::(&VTable::::VTABLE).cast_mut(); + + // SAFETY: `ops` is static, `parent` is bound, and `size` is padded so the allocation made + // by `_fwctl_alloc_device` satisfies the size and alignment required by `Device`. + let raw = unsafe { bindings::_fwctl_alloc_device(parent.as_raw(), ops, size) }; + let this = NonNull::new(raw.cast::()).ok_or(ENOMEM)?; + + // INVARIANT: Set `registration_data` to dangling (no registration yet). + // SAFETY: `this` points to the allocation just returned by fwctl. + unsafe { + (&raw mut (*this.as_ptr()).registration_data) + .write(UnsafeCell::new(NonNull::dangling())); + }; + + // SAFETY: `this` owns the initial reference. + Ok(unsafe { ARef::from_raw(this) }) + } + + /// Returns the underlying `fwctl_device` pointer. + #[inline] + fn as_raw(&self) -> *mut bindings::fwctl_device { + self.dev.get() + } + + /// Borrows a Rust fwctl device from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a valid `fwctl_device` embedded in a [`Device`]. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_device) -> &'a Self { + // SAFETY: The caller upholds the offset-0 `Device` invariant. + unsafe { &*ptr.cast() } + } + + /// Invokes `f` with the registration data. + /// + /// The higher-ranked callback prevents the erased registration lifetime from escaping and + /// permits registration data that is invariant over its lifetime parameter. + /// + /// # Safety + /// + /// The caller must ensure that the device is registered and that this is called from a fwctl + /// callback protected by `registration_lock`. + #[inline] + unsafe fn with_registration_data( + &self, + f: impl for<'a> FnOnce(&Device, &'a T::RegistrationData<'a>) -> R, + ) -> R { + // SAFETY: Caller guarantees the device is registered, so the pointer is valid. + // Lifetimes do not affect layout. The higher-ranked callback prevents the shortened + // lifetime from escaping or being selected by the caller. + let reg_data = unsafe { + (*self.registration_data.get()) + .cast::>() + .as_ref() + }; + + f(self, reg_data) + } +} + +impl AsRef for Device { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: `self` contains a live fwctl_device. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + // SAFETY: The embedded device is initialised by fwctl. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: `fwctl_get` increments the refcount of a valid fwctl_device. +// `fwctl_put` decrements it and frees the device when it reaches zero. +unsafe impl AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self` holds a live reference. + unsafe { bindings::fwctl_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller owns a live reference. + unsafe { bindings::fwctl_put(obj.cast().as_ptr()) }; + } +} + +// SAFETY: `Device` is refcounted by the fwctl core and may be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: Shared access to the embedded `fwctl_device` is protected by the fwctl core. The +// `registration_data` field is only mutated before registration and after unregistration (both +// single-threaded with respect to callbacks). +unsafe impl Sync for Device {} + +/// A registered fwctl device. +/// +/// Owns the [`RegistrationData`](Operations::RegistrationData) made available to driver callbacks. +/// The parent device lifetime ensures that [`fwctl_unregister`] runs before the parent driver +/// unbinds. +/// +/// On drop the device is unregistered (all user contexts are closed and `ops` is set to `NULL`) +/// and the registration data is dropped. +/// +/// [`fwctl_unregister`]: srctree/drivers/fwctl/main.c +pub struct Registration<'a, T: Operations> { + dev: ARef>, + _reg_data: Pin>>, +} + +impl<'a, T: Operations> Registration<'a, T> { + /// Register a previously allocated fwctl device with the given registration data. + /// + /// The `reg_data` is owned by the registration and accessible during callbacks. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since `fwctl_unregister` must be called before the + /// parent device is unbound. + /// + /// `dev` must be an unregistered [`Device`] that is not associated with any live + /// [`Registration`], and no other thread may attempt to register the same device concurrently. + pub unsafe fn new( + parent: &'a device::Device, + dev: &Device, + reg_data: impl PinInit, Error>, + ) -> Result { + let actual_parent = dev.as_ref().parent().ok_or(EINVAL)?; + let parent_device: &device::Device = parent; + if !core::ptr::eq(actual_parent, parent_device) { + return Err(EINVAL); + } + + let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once callbacks can be invoked. The `'static` type is only an erased storage + // handle; callbacks access the pointer through a higher-ranked closure. + let ptr: NonNull> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *dev.registration_data.get() = ptr }; + + // SAFETY: `dev` is a valid fwctl_device backed by an ARef. + let ret = unsafe { bindings::fwctl_register(dev.as_raw()) }; + if ret != 0 { + // SAFETY: No concurrent readers; registration failed. + unsafe { *dev.registration_data.get() = NonNull::dangling() }; + return Err(Error::from_errno(ret)); + } + + Ok(Self { + dev: dev.into(), + _reg_data: reg_data, + }) + } +} + +impl Drop for Registration<'_, T> { + fn drop(&mut self) { + // SAFETY: The Registration lifetime guarantees that the parent device is still bound. + // `fwctl_unregister` takes the write lock, closes all user contexts, and sets ops=NULL. + // After it returns, no callbacks can be running or will run. + unsafe { bindings::fwctl_unregister(self.dev.as_raw()) }; + + // SAFETY: `fwctl_unregister` guarantees no concurrent readers. + unsafe { *self.dev.registration_data.get() = NonNull::dangling() }; + + // `self._reg_data` is dropped here, after callbacks have stopped. + } +} + +/// Internal per-FD user context wrapping `struct fwctl_uctx` and `T`. +/// +/// Not exposed to drivers; they work with `&T` / `Pin<&mut T>` directly. +#[repr(C)] +#[pin_data] +struct UserCtx { + #[pin] + fwctl_uctx: Opaque, + #[pin] + uctx: T, +} + +impl UserCtx { + /// Borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx` that remains + /// valid and does not move for the duration of `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a Self> { + // SAFETY: The caller upholds the `UserCtx` embedding, lifetime, and pinning invariants. + unsafe { Pin::new_unchecked(&*container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx)) } + } + + /// Mutably borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// - `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx` that remains + /// valid and does not move for the duration of `'a`. + /// - The caller must ensure exclusive access to the `UserCtx` for the duration of `'a`. + #[inline] + unsafe fn from_raw_mut<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a mut Self> { + // SAFETY: The caller upholds the embedding, lifetime, pinning, and exclusivity invariants. + unsafe { + Pin::new_unchecked( + &mut *container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx).cast_mut(), + ) + } + } + + /// Returns a reference to the fwctl [`Device`] that owns this context. + #[inline] + fn device(self: Pin<&Self>) -> &Device { + // SAFETY: fwctl initialises this pointer before any driver callback. + let raw_fwctl = unsafe { (*self.fwctl_uctx.get()).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device` layout. + unsafe { Device::from_raw(raw_fwctl) } + } + + /// Returns a pinned reference to the driver context. + #[inline] + fn uctx(self: Pin<&Self>) -> Pin<&T> { + ::pin_init::assert_pinned!(UserCtx, uctx, T, inline); + + // SAFETY: `uctx` is structurally pinned. + unsafe { self.map_unchecked(|ctx| &ctx.uctx) } + } +} + +/// Static vtable mapping Rust trait methods to C callbacks. +struct VTable(PhantomData); + +impl VTable { + /// The fwctl operations vtable for this driver type. + const VTABLE: bindings::fwctl_ops = bindings::fwctl_ops { + // CAST: `DeviceType` has the same `u32` representation as the C enum field. + device_type: T::DEVICE_TYPE as u32, + uctx_size: kmalloc_aligned_size::>(), + open_uctx: Some(Self::open_uctx_callback), + close_uctx: Some(Self::close_uctx_callback), + info: Some(Self::info_callback), + fw_rpc: Some(Self::fw_rpc_callback), + }; + + /// Initialises a newly opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must be a valid `fwctl_uctx` embedded in a `UserCtx` with + /// sufficient allocated space for the uctx field. + unsafe extern "C" fn open_uctx_callback(uctx: *mut bindings::fwctl_uctx) -> ffi::c_int { + const_assert!( + core::mem::offset_of!(UserCtx, fwctl_uctx) == 0, + "struct fwctl_uctx must be at offset 0" + ); + + // SAFETY: fwctl sets this pointer before calling `open_uctx`. + let raw_fwctl = unsafe { (*uctx).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device` layout. + let device = unsafe { Device::::from_raw(raw_fwctl) }; + + let uctx_offset = core::mem::offset_of!(UserCtx, uctx); + // SAFETY: `uctx_size` reserves space for the full `UserCtx`. + let uctx_ptr: *mut T = unsafe { uctx.byte_add(uctx_offset).cast() }; + + // SAFETY: `open_uctx` is called under `registration_lock` read, so the device is + // registered. `uctx_ptr` addresses the uninitialised pinned context reserved by + // `uctx_size`. + unsafe { + device.with_registration_data(|device, reg_data| { + match pin_init::raw_try_init(uctx_ptr, T::open(device, reg_data)) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + }) + } + } + + /// Closes and drops an opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must point to a fully initialised `UserCtx`. + unsafe extern "C" fn close_uctx_callback(uctx: *mut bindings::fwctl_uctx) { + // SAFETY: fwctl keeps the owning device live for this callback. + let device = unsafe { Device::::from_raw((*uctx).fwctl) }; + + // SAFETY: close is called for an opened Rust user context. + let mut ctx = unsafe { UserCtx::::from_raw_mut(uctx) }; + + // SAFETY: `close_uctx` is called under `registration_lock` write (from + // `fwctl_unregister`) or read (from `fwctl_fops_release`), so the device is registered. + unsafe { + device.with_registration_data(|device, reg_data| { + T::close(ctx.as_mut().project().uctx, device, reg_data); + }); + } + + // SAFETY: close is the last callback before fwctl frees the allocation. + unsafe { core::ptr::drop_in_place(ctx.project().uctx.get_unchecked_mut()) }; + } + + /// Returns device-specific information for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx`. + /// - `length` must be a valid pointer. + unsafe extern "C" fn info_callback( + uctx: *mut bindings::fwctl_uctx, + length: *mut usize, + ) -> *mut ffi::c_void { + // SAFETY: info is called for an opened Rust user context. + let ctx = unsafe { UserCtx::::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: `info` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| T::info(ctx.uctx(), device, reg_data)) + }; + + match result { + Ok(kvec) if kvec.is_empty() => { + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = 0 }; + // Return NULL for empty data; kfree(NULL) is safe. + core::ptr::null_mut() + } + Ok(kvec) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = len }; + ptr.cast::() + } + Err(e) => Error::to_ptr(e), + } + } + + /// Dispatches a firmware RPC for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx`. + /// - `rpc_in` must be valid, initialised, and exclusively accessible for `in_len` bytes. + /// - `out_len` must be valid for reading and writing an initialised `usize`. + unsafe extern "C" fn fw_rpc_callback( + uctx: *mut bindings::fwctl_uctx, + scope: u32, + rpc_in: *mut ffi::c_void, + in_len: usize, + out_len: *mut usize, + ) -> *mut ffi::c_void { + let scope = match RpcScope::try_from(scope) { + Ok(s) => s, + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` points to an initialised `usize` supplied by fwctl. + let max_output_len = unsafe { *out_len }; + + // SAFETY: RPC is called for an opened Rust user context. + let ctx = unsafe { UserCtx::::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: fwctl passes an exclusively owned buffer that is valid and initialised for + // `in_len` bytes. It remains live for the duration of this callback. + let rpc_buf = unsafe { slice::from_raw_parts_mut(rpc_in.cast::(), in_len) }; + + // SAFETY: `fw_rpc` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| { + T::fw_rpc(ctx.uctx(), device, reg_data, scope, rpc_buf, max_output_len) + }) + }; + + let (response, response_len) = match result { + Ok(FwRpcResponse::InPlace(len)) => { + if len > in_len { + return Error::to_ptr(EINVAL); + } + + (rpc_in, len) + } + Ok(FwRpcResponse::NewBuffer(kvec)) if kvec.is_empty() => { + // Return NULL for empty data; kvfree(NULL) is safe. + (core::ptr::null_mut(), 0) + } + Ok(FwRpcResponse::NewBuffer(kvec)) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + (ptr.cast::(), len) + } + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` is a valid out-parameter. + unsafe { *out_len = response_len }; + response + } +} diff --git a/rust/kernel/generated_arch_reachable_asm.rs.S b/rust/kernel/generated_arch_reachable_asm.rs.S new file mode 100644 index 000000000..3886a9ad3 --- /dev/null +++ b/rust/kernel/generated_arch_reachable_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_REACHABLE) diff --git a/rust/kernel/generated_arch_static_branch_asm.rs.S b/rust/kernel/generated_arch_static_branch_asm.rs.S new file mode 100644 index 000000000..2afb63870 --- /dev/null +++ b/rust/kernel/generated_arch_static_branch_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_STATIC_BRANCH_ASM("{symb} + {off} + {branch}", "{l_yes}")) diff --git a/rust/kernel/generated_arch_warn_asm.rs.S b/rust/kernel/generated_arch_warn_asm.rs.S new file mode 100644 index 000000000..409eb4c2d --- /dev/null +++ b/rust/kernel/generated_arch_warn_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_ASM("{file}", "{line}", "{flags}", "{size}")) diff --git a/rust/kernel/gpu.rs b/rust/kernel/gpu.rs new file mode 100644 index 000000000..1dc5d0c8c --- /dev/null +++ b/rust/kernel/gpu.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU subsystem abstractions. + +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod buddy; diff --git a/rust/kernel/gpu/buddy.rs b/rust/kernel/gpu/buddy.rs new file mode 100644 index 000000000..d502ada6e --- /dev/null +++ b/rust/kernel/gpu/buddy.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU buddy allocator bindings. +//! +//! C header: [`include/linux/gpu_buddy.h`](srctree/include/linux/gpu_buddy.h) +//! +//! This module provides Rust abstractions over the Linux kernel's GPU buddy +//! allocator, which implements a binary buddy memory allocator. +//! +//! The buddy allocator manages a contiguous address space and allocates blocks +//! in power-of-two sizes, useful for GPU physical memory management. +//! +//! # Examples +//! +//! Create a buddy allocator and perform a basic range allocation: +//! +//! ``` +//! use kernel::{ +//! gpu::buddy::{ +//! GpuBuddy, +//! GpuBuddyAllocFlags, +//! GpuBuddyAllocMode, +//! GpuBuddyParams, // +//! }, +//! prelude::*, +//! ptr::Alignment, +//! sizes::*, // +//! }; +//! +//! // Create a 1GB buddy allocator with 4KB minimum chunk size. +//! let buddy = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_1G as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! assert_eq!(buddy.size(), SZ_1G as u64); +//! assert_eq!(buddy.chunk_size(), Alignment::new::()); +//! let initial_free = buddy.avail(); +//! +//! // Allocate 16MB. Block lands at the top of the address range. +//! let allocated = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = allocated.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); // 2^12 pages = 16MB +//! assert_eq!(block.size(), SZ_16M as u64); +//! assert_eq!(allocated.iter().count(), 1); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(allocated); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Top-down allocation allocates from the highest addresses: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{GpuBuddy, GpuBuddyAllocMode, GpuBuddyAllocFlags, GpuBuddyParams}, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! let topdown = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::TopDown, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = topdown.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); +//! assert_eq!(block.size(), SZ_16M as u64); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(topdown); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Non-contiguous allocation can fill fragmented memory by returning multiple +//! blocks: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! // Create fragmentation by allocating 4MB blocks at [0,4M) and [8M,12M). +//! let frag1 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_4M as u64); +//! +//! let frag2 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_8M as u64); +//! +//! // Allocate 8MB, this returns 2 blocks from the holes. +//! let fragmented = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_16M as u64), +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let (mut count, mut total) = (0u32, 0u64); +//! for block in fragmented.iter() { +//! assert_eq!(block.size(), SZ_4M as u64); +//! total += block.size(); +//! count += 1; +//! } +//! assert_eq!(total, SZ_8M as u64); +//! assert_eq!(count, 2); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Contiguous allocation fails when only fragmented space is available: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlag, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! // Create a small 16MB buddy allocator with fragmented memory. +//! let small = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_16M as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! let _hole1 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! let _hole2 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! // 8MB contiguous should fail, only two non-contiguous 4MB holes exist. +//! let result = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlag::Contiguous, +//! ), +//! GFP_KERNEL, +//! ); +//! assert!(result.is_err()); +//! # Ok::<(), Error>(()) +//! ``` + +use core::ops::Range; + +use crate::{ + bindings, + clist_create, + error::to_result, + interop::list::CListHead, + new_mutex, + prelude::*, + ptr::Alignment, + sync::{ + lock::mutex::MutexGuard, + Arc, + Mutex, // + }, + types::Opaque, // +}; + +/// Allocation mode for the GPU buddy allocator. +/// +/// The mode determines the primary allocation strategy. Modes are mutually +/// exclusive: an allocation is either simple, range-constrained, or top-down. +/// +/// Orthogonal modifier flags (e.g., contiguous, clear) are specified separately +/// via [`GpuBuddyAllocFlags`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GpuBuddyAllocMode { + /// Simple allocation without constraints. + Simple, + /// Range-based allocation within the given address range. + Range(Range), + /// Allocate from top of address space downward. + TopDown, +} + +impl GpuBuddyAllocMode { + /// Returns the C flags corresponding to the allocation mode. + fn as_flags(&self) -> usize { + match self { + Self::Simple => 0, + Self::Range(_) => bindings::GPU_BUDDY_RANGE_ALLOCATION, + Self::TopDown => bindings::GPU_BUDDY_TOPDOWN_ALLOCATION, + } + } + + /// Extracts the range start/end, defaulting to `(0, 0)` for non-range modes. + fn range(&self) -> (u64, u64) { + match self { + Self::Range(range) => (range.start, range.end), + _ => (0, 0), + } + } +} + +crate::impl_flags!( + /// Modifier flags for GPU buddy allocation. + /// + /// These flags can be combined with any [`GpuBuddyAllocMode`] to control + /// additional allocation behavior. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + pub struct GpuBuddyAllocFlags(usize); + + /// Individual modifier flag for GPU buddy allocation. + #[derive(Clone, Copy, PartialEq, Eq)] + pub enum GpuBuddyAllocFlag { + /// Allocate physically contiguous blocks. + Contiguous = bindings::GPU_BUDDY_CONTIGUOUS_ALLOCATION, + + /// Request allocation from cleared (zeroed) memory. + Clear = bindings::GPU_BUDDY_CLEAR_ALLOCATION, + + /// Disable trimming of partially used blocks. + TrimDisable = bindings::GPU_BUDDY_TRIM_DISABLE, + } +); + +/// Parameters for creating a GPU buddy allocator. +pub struct GpuBuddyParams { + /// Base offset (in bytes) where the managed memory region starts. + /// Allocations will be offset by this value. + pub base_offset: u64, + /// Total size (in bytes) of the address space managed by the allocator. + pub size: u64, + /// Minimum allocation unit / chunk size; must be >= 4KB. + pub chunk_size: Alignment, +} + +/// Inner structure holding the actual buddy allocator. +/// +/// # Synchronization +/// +/// The C `gpu_buddy` API requires synchronization (see `include/linux/gpu_buddy.h`). +/// Internal locking ensures all allocator and free operations are properly +/// synchronized, preventing races between concurrent allocations and the +/// freeing that occurs when [`AllocatedBlocks`] is dropped. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains an initialized buddy allocator. +#[pin_data(PinnedDrop)] +struct GpuBuddyInner { + #[pin] + inner: Opaque, + + // TODO: Replace `Mutex<()>` with `Mutex>` once `Mutex::new()` + // accepts `impl PinInit`. + #[pin] + lock: Mutex<()>, + /// Cached creation parameters (do not change after init). + params: GpuBuddyParams, +} + +impl GpuBuddyInner { + /// Create a pin-initializer for the buddy allocator. + fn new(params: GpuBuddyParams) -> impl PinInit { + let size = params.size; + let chunk_size = params.chunk_size; + + // INVARIANT: `gpu_buddy_init` returns 0 on success, at which point the + // `gpu_buddy` structure is initialized and ready for use with all + // `gpu_buddy_*` APIs. `try_pin_init!` only completes if all fields succeed, + // so the invariant holds when construction finishes. + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `ptr` points to valid uninitialized memory from the pin-init + // infrastructure. `gpu_buddy_init` will initialize the structure. + to_result(unsafe { + bindings::gpu_buddy_init(ptr, size, chunk_size.as_usize() as u64) + }) + }), + lock <- new_mutex!(()), + params, + }) + } + + /// Lock the mutex and return a guard for accessing the allocator. + fn lock(&self) -> GpuBuddyGuard<'_> { + GpuBuddyGuard { + inner: self, + _guard: self.lock.lock(), + } + } +} + +#[pinned_drop] +impl PinnedDrop for GpuBuddyInner { + fn drop(self: Pin<&mut Self>) { + let guard = self.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + unsafe { bindings::gpu_buddy_fini(guard.as_raw()) }; + } +} + +// SAFETY: `GpuBuddyInner` can be sent between threads. +unsafe impl Send for GpuBuddyInner {} + +// SAFETY: `GpuBuddyInner` is `Sync` because `GpuBuddyInner::lock` +// serializes all access to the C allocator, preventing data races. +unsafe impl Sync for GpuBuddyInner {} + +/// Guard that proves the lock is held, enabling access to the allocator. +/// +/// The `_guard` holds the lock for the duration of this guard's lifetime. +struct GpuBuddyGuard<'a> { + inner: &'a GpuBuddyInner, + _guard: MutexGuard<'a, ()>, +} + +impl GpuBuddyGuard<'_> { + /// Get a raw pointer to the underlying C `gpu_buddy` structure. + fn as_raw(&self) -> *mut bindings::gpu_buddy { + self.inner.inner.get() + } +} + +/// GPU buddy allocator instance. +/// +/// This structure wraps the C `gpu_buddy` allocator using reference counting. +/// The allocator is automatically cleaned up when all references are dropped. +/// +/// Refer to the module-level documentation for usage examples. +pub struct GpuBuddy(Arc); + +impl GpuBuddy { + /// Create a new buddy allocator. + /// + /// The allocator manages a contiguous address space of the given size, with the + /// specified minimum allocation unit (chunk_size must be at least 4KB). + pub fn new(params: GpuBuddyParams) -> Result { + Arc::pin_init(GpuBuddyInner::new(params), GFP_KERNEL).map(Self) + } + + /// Get the base offset for allocations. + pub fn base_offset(&self) -> u64 { + self.0.params.base_offset + } + + /// Get the chunk size (minimum allocation unit). + pub fn chunk_size(&self) -> Alignment { + self.0.params.chunk_size + } + + /// Get the total managed size. + pub fn size(&self) -> u64 { + self.0.params.size + } + + /// Get the available (free) memory in bytes. + pub fn avail(&self) -> u64 { + let guard = self.0.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized allocator. + // `guard` provides exclusive access. + unsafe { (*guard.as_raw()).avail } + } + + /// Allocate blocks from the buddy allocator. + /// + /// Returns a pin-initializer for [`AllocatedBlocks`]. + pub fn alloc_blocks( + &self, + mode: GpuBuddyAllocMode, + size: u64, + min_block_size: Alignment, + flags: impl Into, + ) -> impl PinInit { + let buddy_arc = Arc::clone(&self.0); + let (start, end) = mode.range(); + let mode_flags = mode.as_flags(); + let modifier_flags = flags.into(); + + // Create pin-initializer that initializes list and allocates blocks. + try_pin_init!(AllocatedBlocks { + buddy: buddy_arc, + list <- CListHead::new(), + _: { + // Reject zero-sized or inverted ranges. + if let GpuBuddyAllocMode::Range(range) = &mode { + if range.is_empty() { + Err::<(), Error>(EINVAL)?; + } + } + + // Lock while allocating to serialize with concurrent frees. + let guard = buddy.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + to_result(unsafe { + bindings::gpu_buddy_alloc_blocks( + guard.as_raw(), + start, + end, + size, + min_block_size.as_usize() as u64, + list.as_raw(), + mode_flags | usize::from(modifier_flags), + ) + })? + } + }) + } +} + +/// Allocated blocks from the buddy allocator with automatic cleanup. +/// +/// This structure owns a list of allocated blocks and ensures they are +/// automatically freed when dropped. Use `iter()` to iterate over all +/// allocated blocks. +/// +/// # Invariants +/// +/// - `list` is an initialized, valid list head containing allocated blocks. +#[pin_data(PinnedDrop)] +pub struct AllocatedBlocks { + #[pin] + list: CListHead, + buddy: Arc, +} + +impl AllocatedBlocks { + /// Check if the block list is empty. + pub fn is_empty(&self) -> bool { + // An empty list head points to itself. + !self.list.is_linked() + } + + /// Iterate over allocated blocks. + /// + /// Returns an iterator yielding [`AllocatedBlock`] values. Each [`AllocatedBlock`] + /// borrows `self` and is only valid for the duration of that borrow. + pub fn iter(&self) -> impl Iterator> + '_ { + let head = self.list.as_raw(); + // SAFETY: Per the type invariant, `list` is an initialized sentinel `list_head` + // and is not concurrently modified (we hold a `&self` borrow). The list contains + // `gpu_buddy_block` items linked via `__bindgen_anon_1.link`. `Block` is + // `#[repr(transparent)]` over `gpu_buddy_block`. + let clist = unsafe { + clist_create!( + head, + Block, + bindings::gpu_buddy_block, + __bindgen_anon_1.link + ) + }; + + clist + .iter() + .map(|this| AllocatedBlock { this, blocks: self }) + } +} + +#[pinned_drop] +impl PinnedDrop for AllocatedBlocks { + fn drop(self: Pin<&mut Self>) { + let guard = self.buddy.lock(); + + // SAFETY: + // - list is valid per the type's invariants. + // - guard provides exclusive access to the allocator. + unsafe { + bindings::gpu_buddy_free_list(guard.as_raw(), self.list.as_raw(), 0); + } + } +} + +/// A GPU buddy block. +/// +/// Transparent wrapper over C `gpu_buddy_block` structure. This type is returned +/// as references during iteration over [`AllocatedBlocks`]. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains a valid, allocated `gpu_buddy_block`. +#[repr(transparent)] +struct Block(Opaque); + +impl Block { + /// Get a raw pointer to the underlying C block. + fn as_raw(&self) -> *mut bindings::gpu_buddy_block { + self.0.get() + } + + /// Get the block's raw offset in the buddy address space (without base offset). + fn offset(&self) -> u64 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_offset(self.as_raw()) } + } + + /// Get the block order. + fn order(&self) -> u32 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_order(self.as_raw()) } + } +} + +// SAFETY: `Block` is a wrapper around `gpu_buddy_block` which can be +// sent across threads safely. +unsafe impl Send for Block {} + +// SAFETY: `Block` is only accessed through shared references after +// allocation, and thus safe to access concurrently across threads. +unsafe impl Sync for Block {} + +/// A buddy block paired with its owning [`AllocatedBlocks`] context. +/// +/// Unlike a raw block, which only knows its offset within the buddy address +/// space, an [`AllocatedBlock`] also has access to the allocator's `base_offset` +/// and `chunk_size`, enabling it to compute absolute offsets and byte sizes. +/// +/// Returned by [`AllocatedBlocks::iter()`]. +pub struct AllocatedBlock<'a> { + this: &'a Block, + blocks: &'a AllocatedBlocks, +} + +impl AllocatedBlock<'_> { + /// Get the block's offset in the address space. + /// + /// Returns the absolute offset including the allocator's base offset. + /// This is the actual address to use for accessing the allocated memory. + pub fn offset(&self) -> u64 { + self.blocks.buddy.params.base_offset + self.this.offset() + } + + /// Get the block order (size = chunk_size << order). + pub fn order(&self) -> u32 { + self.this.order() + } + + /// Get the block's size in bytes. + pub fn size(&self) -> u64 { + (self.blocks.buddy.params.chunk_size.as_usize() as u64) << self.this.order() + } +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs new file mode 100644 index 000000000..0487bae81 --- /dev/null +++ b/rust/kernel/i2c.rs @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! I2C Driver subsystem + +// I2C Driver abstractions. +use crate::{ + acpi, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + devres::Devres, + driver, + error::*, + of, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::{ + from_ref, + NonNull, // + }, // +}; + +/// An I2C device id table. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::i2c_device_id); + +impl DeviceId { + const I2C_NAME_SIZE: usize = 20; + + /// Create a new device id from an I2C 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_NAME_SIZE, "ID exceeds 20 bytes"); + let mut i2c: bindings::i2c_device_id = pin_init::zeroed(); + let mut i = 0; + while i < src.len() { + i2c.name[i] = src[i]; + i += 1; + } + + Self(i2c) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `i2c_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::i2c_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); +} + +/// IdTable type for I2C +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a I2C `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! i2c_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("i2c", $crate::i2c::DeviceId, $($tt)*); + }; +} + +/// An adapter for the registration of I2C drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::i2c_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct i2c_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::i2c_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + idrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + build_assert!( + T::ACPI_ID_TABLE.is_some() || T::OF_ID_TABLE.is_some() || T::I2C_ID_TABLE.is_some(), + "At least one of ACPI/OF/Legacy tables must be present when registering an i2c driver" + ); + + let i2c_table = match T::I2C_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct i2c_client` on initialization. + unsafe { + (*idrv.get()).driver.name = name.as_char_ptr(); + (*idrv.get()).probe = Some(Self::probe_callback); + (*idrv.get()).remove = Some(Self::remove_callback); + (*idrv.get()).shutdown = Some(Self::shutdown_callback); + (*idrv.get()).id_table = i2c_table; + (*idrv.get()).driver.of_match_table = of_table; + (*idrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::i2c_register_driver(module.as_ptr(), idrv.get()) }) + } + + unsafe fn unregister(idrv: &Opaque) { + // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::i2c_del_driver(idrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_int { + // SAFETY: The I2C bus only ever calls the probe callback with a valid pointer to a + // `struct i2c_client`. + // + // INVARIANT: `idev` is valid for the duration of `probe_callback()`. + let idev = unsafe { &*idev.cast::>>() }; + + let info = Self::i2c_id_info(idev).or_else(|| { + // SAFETY: `idev` matched data is of type `Self::IdInfo`. + unsafe { ::id_info(idev.as_ref()) } + }); + + from_result(|| { + let data = T::probe(idev, info); + + idev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + let idev = unsafe { &*idev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; + + T::unbind(idev, data); + } + + extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `shutdown_callback` is only ever called for a valid `idev` + let idev = unsafe { &*idev.cast::>>() }; + + // SAFETY: `shutdown_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; + + T::shutdown(idev, data); + } + + /// The [`i2c::IdTable`] of the corresponding driver. + fn i2c_id_table() -> Option::IdInfo>> { + T::I2C_ID_TABLE + } + + /// Returns the driver's private data from the matching entry in the [`i2c::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`i2c::IdTable`]. + fn i2c_id_info(dev: &I2cClient) -> Option<&'static ::IdInfo> { + let table = Self::i2c_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for reads + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::i2c_match_id(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + return None; + } + + // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct i2c_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::() }; + + // SAFETY: `id` comes from `table` which is of type `IdArray<_, Self::IdInfo>`. + Some(unsafe { id.info_unchecked::() }) + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single i2c driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_i2c_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_i2c_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::i2c::Adapter, { $($f)* }); + }; +} + +/// The i2c driver trait. +/// +/// Drivers must implement this trait in order to get a i2c driver registered. +/// +/// # Example +/// +///``` +/// # use kernel::{acpi, bindings, device::Core, i2c, of}; +/// +/// struct MyDriver; +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// kernel::i2c_device_table!( +/// I2C_TABLE, +/// ::IdInfo, +/// [ +/// (i2c::DeviceId::new(c"rust_driver_i2c"), ()) +/// ] +/// ); +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// impl i2c::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// +/// fn shutdown<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// this: Pin<&Self::Data<'bound>>, +/// ) { +/// } +/// } +///``` +pub trait Driver { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const I2C_ID_TABLE: Option> = None; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// I2C driver probe. + /// + /// Called when a new i2c client is added or discovered. + /// Implementers should attempt to initialize the client here. + fn probe<'bound>( + dev: &'bound I2cClient>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// I2C driver shutdown. + /// + /// Called by the kernel during system reboot or power-off to allow the [`Driver`] to bring the + /// [`I2cClient`] into a safe state. Implementing this callback is optional. + /// + /// Typical actions include stopping transfers, disabling interrupts, or resetting the hardware + /// to prevent undesired behavior during shutdown. + /// + /// This callback is distinct from final resource cleanup, as the driver instance remains valid + /// after it returns. Any deallocation or teardown of driver-owned resources should instead be + /// handled in `Drop`. + fn shutdown<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } + + /// I2C driver unbind. + /// + /// Called when the [`I2cClient`] is unbound from its bound [`Driver`]. Implementing this + /// callback is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The i2c adapter representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_adapter`. The +/// implementation abstracts the usage of an existing C `struct i2c_adapter` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cAdapter`] instance represents a valid `struct i2c_adapter` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cAdapter( + Opaque, + PhantomData, +); + +impl I2cAdapter { + fn as_raw(&self) -> *mut bindings::i2c_adapter { + self.0.get() + } +} + +impl I2cAdapter { + /// Returns the I2C Adapter index. + #[inline] + pub fn index(&self) -> i32 { + // SAFETY: `self.as_raw` is a valid pointer to a `struct i2c_adapter`. + unsafe { (*self.as_raw()).nr } + } + + /// Gets pointer to an `i2c_adapter` by index. + #[inline] + pub fn get(index: i32) -> Result> { + // SAFETY: `index` must refer to a valid I2C adapter; the kernel + // guarantees that `i2c_get_adapter(index)` returns either a valid + // pointer or NULL. `NonNull::new` guarantees the correct check. + let adapter = NonNull::new(unsafe { bindings::i2c_get_adapter(index) }).ok_or(ENODEV)?; + + // SAFETY: `adapter` is non-null and points to a live `i2c_adapter`. + // `I2cAdapter` is #[repr(transparent)], so this cast is valid. + // `i2c_get_adapter` returned the adapter with an incremented refcount, which we pass to + // the `ARef`. + Ok(unsafe { ARef::from_raw(adapter.cast::>()) }) + } +} + +// SAFETY: `I2cAdapter` is a transparent wrapper of a type that doesn't depend on +// `I2cAdapter`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cAdapter }); +kernel::impl_device_context_into_aref!(I2cAdapter); + +// SAFETY: Instances of `I2cAdapter` are always reference-counted. +unsafe impl AlwaysRefCounted for I2cAdapter { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::i2c_get_adapter(self.index()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } + } +} + +/// The i2c board info representation +/// +/// This structure represents the Rust abstraction for a C `struct i2c_board_info` structure, +/// which is used for manual I2C client creation. +#[repr(transparent)] +pub struct I2cBoardInfo(bindings::i2c_board_info); + +impl I2cBoardInfo { + const I2C_TYPE_SIZE: usize = 20; + /// Create a new [`I2cBoardInfo`] for a kernel driver. + #[inline(always)] + pub const fn new(type_: &'static CStr, addr: u16) -> Self { + let src = type_.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_TYPE_SIZE, "Type exceeds 20 bytes"); + let mut i2c_board_info: bindings::i2c_board_info = pin_init::zeroed(); + let mut i: usize = 0; + while i < src.len() { + i2c_board_info.type_[i] = src[i]; + i += 1; + } + + i2c_board_info.addr = addr; + Self(i2c_board_info) + } + + fn as_raw(&self) -> *const bindings::i2c_board_info { + from_ref(&self.0) + } +} + +/// The i2c client representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_client`. The +/// implementation abstracts the usage of an existing C `struct i2c_client` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cClient`] instance represents a valid `struct i2c_client` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cClient( + Opaque, + PhantomData, +); + +impl I2cClient { + fn as_raw(&self) -> *mut bindings::i2c_client { + self.0.get() + } +} + +// SAFETY: `I2cClient` is a transparent wrapper of `struct i2c_client`. +// The offset is guaranteed to point to a valid device field inside `I2cClient`. +unsafe impl device::AsBusDevice for I2cClient { + const OFFSET: usize = offset_of!(bindings::i2c_client, dev); +} + +// SAFETY: `I2cClient` is a transparent wrapper of a type that doesn't depend on +// `I2cClient`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cClient }); +kernel::impl_device_context_into_aref!(I2cClient); + +// SAFETY: Instances of `I2cClient` are always reference-counted. +unsafe impl AlwaysRefCounted for I2cClient { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(&raw mut (*obj.as_ref().as_raw()).dev) } + } +} + +impl AsRef> for I2cClient { + fn as_ref(&self) -> &device::Device { + let raw = self.as_raw(); + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct i2c_client`. + let dev = unsafe { &raw mut (*raw).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &I2cClient { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if unsafe { bindings::i2c_verify_client(dev.as_raw()).is_null() } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the type of `dev` equals to + // `bindings::i2c_client_type`, hence `dev` must be embedded in a valid + // `struct i2c_client` as guaranteed by the corresponding C code. + let idev = unsafe { container_of!(dev.as_raw(), bindings::i2c_client, dev) }; + + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + Ok(unsafe { &*idev.cast() }) + } +} + +// SAFETY: A `I2cClient` is always reference-counted and can be released from any thread. +unsafe impl Send for I2cClient {} + +// SAFETY: `I2cClient` can be shared among threads because all methods of `I2cClient` +// (i.e. `I2cClient) are thread safe. +unsafe impl Sync for I2cClient {} + +/// The registration of an i2c client device. +/// +/// This type represents the registration of a [`struct i2c_client`]. When an instance of this +/// type is dropped, its respective i2c client device will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.0` always holds a valid pointer to an initialized and registered +/// [`struct i2c_client`]. +#[repr(transparent)] +pub struct Registration(NonNull); + +impl Registration { + /// The C `i2c_new_client_device` function wrapper for manual I2C client creation. + pub fn new<'a>( + i2c_adapter: &I2cAdapter, + i2c_board_info: &I2cBoardInfo, + parent_dev: &'a device::Device, + ) -> impl PinInit, Error> + 'a { + Devres::new(parent_dev, Self::try_new(i2c_adapter, i2c_board_info)) + } + + fn try_new(i2c_adapter: &I2cAdapter, i2c_board_info: &I2cBoardInfo) -> Result { + // SAFETY: the kernel guarantees that `i2c_new_client_device()` returns either a valid + // pointer or NULL. `from_err_ptr` separates errors. Following `NonNull::new` + // checks for NULL. + let raw_dev = from_err_ptr(unsafe { + bindings::i2c_new_client_device(i2c_adapter.as_raw(), i2c_board_info.as_raw()) + })?; + + let dev_ptr = NonNull::new(raw_dev).ok_or(ENODEV)?; + + Ok(Self(dev_ptr)) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `Drop` is only called for a valid `Registration`, which by invariant + // always contains a non-null pointer to an `i2c_client`. + unsafe { bindings::i2c_unregister_device(self.0.as_ptr()) } + } +} + +// SAFETY: A `Registration` of a `struct i2c_client` can be released from any thread. +unsafe impl Send for Registration {} + +// SAFETY: `Registration` offers no interior mutability (no mutation through &self +// and no mutable access is exposed) +unsafe impl Sync for Registration {} diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs new file mode 100644 index 000000000..384753fe0 --- /dev/null +++ b/rust/kernel/id_pool.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! Rust API for an ID pool backed by a [`BitmapVec`]. + +use crate::alloc::{AllocError, Flags}; +use crate::bitmap::BitmapVec; + +/// Represents a dynamic ID pool backed by a [`BitmapVec`]. +/// +/// Clients acquire and release IDs from unset bits in a bitmap. +/// +/// The capacity of the ID pool may be adjusted by users as +/// needed. The API supports the scenario where users need precise control +/// over the time of allocation of a new backing bitmap, which may require +/// release of spinlock. +/// Due to concurrent updates, all operations are re-verified to determine +/// if the grow or shrink is sill valid. +/// +/// # Examples +/// +/// Basic usage +/// +/// ``` +/// use kernel::alloc::AllocError; +/// use kernel::id_pool::{IdPool, UnusedId}; +/// +/// let mut pool = IdPool::with_capacity(64, GFP_KERNEL)?; +/// for i in 0..64 { +/// assert_eq!(i, pool.find_unused_id(i).ok_or(ENOSPC)?.acquire()); +/// } +/// +/// pool.release_id(23); +/// assert_eq!(23, pool.find_unused_id(0).ok_or(ENOSPC)?.acquire()); +/// +/// assert!(pool.find_unused_id(0).is_none()); // time to realloc. +/// let resizer = pool.grow_request().ok_or(ENOSPC)?.realloc(GFP_KERNEL)?; +/// pool.grow(resizer); +/// +/// assert_eq!(pool.find_unused_id(0).ok_or(ENOSPC)?.acquire(), 64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Releasing spinlock to grow the pool +/// +/// ```no_run +/// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; +/// use kernel::sync::{new_spinlock, SpinLock}; +/// use kernel::id_pool::IdPool; +/// +/// fn get_id_maybe_realloc(guarded_pool: &SpinLock) -> Result { +/// let mut pool = guarded_pool.lock(); +/// loop { +/// match pool.find_unused_id(0) { +/// Some(index) => return Ok(index.acquire()), +/// None => { +/// let alloc_request = pool.grow_request(); +/// drop(pool); +/// let resizer = alloc_request.ok_or(AllocError)?.realloc(GFP_KERNEL)?; +/// pool = guarded_pool.lock(); +/// pool.grow(resizer) +/// } +/// } +/// } +/// } +/// ``` +pub struct IdPool { + map: BitmapVec, +} + +/// Indicates that an [`IdPool`] should change to a new target size. +pub struct ReallocRequest { + num_ids: usize, +} + +/// Contains a [`BitmapVec`] of a size suitable for reallocating [`IdPool`]. +pub struct PoolResizer { + new: BitmapVec, +} + +impl ReallocRequest { + /// Allocates a new backing [`BitmapVec`] for [`IdPool`]. + /// + /// This method only prepares reallocation and does not complete it. + /// Reallocation will complete after passing the [`PoolResizer`] to the + /// [`IdPool::grow`] or [`IdPool::shrink`] operation, which will check + /// that reallocation still makes sense. + pub fn realloc(&self, flags: Flags) -> Result { + let new = BitmapVec::new(self.num_ids, flags)?; + Ok(PoolResizer { new }) + } +} + +impl IdPool { + /// Constructs a new [`IdPool`]. + /// + /// The pool will have a capacity of [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn new() -> Self { + Self { + map: BitmapVec::new_inline(), + } + } + + /// Constructs a new [`IdPool`] with space for a specific number of bits. + /// + /// A capacity below [`MAX_INLINE_LEN`] is adjusted to [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn with_capacity(num_ids: usize, flags: Flags) -> Result { + let num_ids = usize::max(num_ids, BitmapVec::MAX_INLINE_LEN); + let map = BitmapVec::new(num_ids, flags)?; + Ok(Self { map }) + } + + /// Returns how many IDs this pool can currently have. + #[inline] + pub fn capacity(&self) -> usize { + self.map.len() + } + + /// Returns a [`ReallocRequest`] if the [`IdPool`] can be shrunk, [`None`] otherwise. + /// + /// The capacity of an [`IdPool`] cannot be shrunk below [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// alloc::AllocError, + /// bitmap::BitmapVec, + /// id_pool::{ + /// IdPool, + /// ReallocRequest, + /// }, + /// }; + /// + /// let mut pool = IdPool::with_capacity(1024, GFP_KERNEL)?; + /// let alloc_request = pool.shrink_request().ok_or(AllocError)?; + /// let resizer = alloc_request.realloc(GFP_KERNEL)?; + /// pool.shrink(resizer); + /// assert_eq!(pool.capacity(), BitmapVec::MAX_INLINE_LEN); + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn shrink_request(&self) -> Option { + let cap = self.capacity(); + // Shrinking below `MAX_INLINE_LEN` is never possible. + if cap <= BitmapVec::MAX_INLINE_LEN { + return None; + } + // Determine if the bitmap can shrink based on the position of + // its last set bit. If the bit is within the first quarter of + // the bitmap then shrinking is possible. In this case, the + // bitmap should shrink to half its current size. + let Some(bit) = self.map.last_bit() else { + return Some(ReallocRequest { + num_ids: BitmapVec::MAX_INLINE_LEN, + }); + }; + if bit >= (cap / 4) { + return None; + } + let num_ids = usize::max(BitmapVec::MAX_INLINE_LEN, cap / 2); + Some(ReallocRequest { num_ids }) + } + + /// Shrinks pool by using a new [`BitmapVec`], if still possible. + #[inline] + pub fn shrink(&mut self, mut resizer: PoolResizer) { + // Between request to shrink that led to allocation of `resizer` and now, + // bits may have changed. + // Verify that shrinking is still possible. In case shrinking to + // the size of `resizer` is no longer possible, do nothing, + // drop `resizer` and move on. + let Some(updated) = self.shrink_request() else { + return; + }; + if updated.num_ids > resizer.new.len() { + return; + } + + resizer.new.copy_and_extend(&self.map); + self.map = resizer.new; + } + + /// Returns a [`ReallocRequest`] for growing this [`IdPool`], if possible. + /// + /// The capacity of an [`IdPool`] cannot be grown above [`MAX_LEN`]. + /// + /// [`MAX_LEN`]: BitmapVec::MAX_LEN + #[inline] + pub fn grow_request(&self) -> Option { + let num_ids = self.capacity() * 2; + if num_ids > BitmapVec::MAX_LEN { + return None; + } + Some(ReallocRequest { num_ids }) + } + + /// Grows pool by using a new [`BitmapVec`], if still necessary. + /// + /// The `resizer` arguments has to be obtained by calling [`Self::grow_request`] + /// on this object and performing a [`ReallocRequest::realloc`]. + #[inline] + pub fn grow(&mut self, mut resizer: PoolResizer) { + // Between request to grow that led to allocation of `resizer` and now, + // another thread may have already grown the capacity. + // In this case, do nothing, drop `resizer` and move on. + if resizer.new.len() <= self.capacity() { + return; + } + + resizer.new.copy_and_extend(&self.map); + self.map = resizer.new; + } + + /// Finds an unused ID in the bitmap. + /// + /// Upon success, returns its index. Otherwise, returns [`None`] + /// to indicate that a [`Self::grow_request`] is needed. + #[inline] + #[must_use] + pub fn find_unused_id(&mut self, offset: usize) -> Option> { + // INVARIANT: `next_zero_bit()` returns None or an integer less than `map.len()` + Some(UnusedId { + id: self.map.next_zero_bit(offset)?, + pool: self, + }) + } + + /// Releases an ID. + #[inline] + pub fn release_id(&mut self, id: usize) { + self.map.clear_bit(id); + } +} + +/// Represents an unused id in an [`IdPool`]. +/// +/// # Invariants +/// +/// The value of `id` is less than `pool.map.len()`. +pub struct UnusedId<'pool> { + id: usize, + pool: &'pool mut IdPool, +} + +impl<'pool> UnusedId<'pool> { + /// Get the unused id as an usize. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_usize(&self) -> usize { + self.id + } + + /// Get the unused id as an u32. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_u32(&self) -> u32 { + // CAST: By the type invariants: + // `self.id < pool.map.len() <= BitmapVec::MAX_LEN = i32::MAX`. + self.id as u32 + } + + /// Acquire the unused id. + #[inline] + pub fn acquire(self) -> usize { + self.pool.map.set_bit(self.id); + self.id + } +} + +impl Default for IdPool { + #[inline] + fn default() -> Self { + Self::new() + } +} diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs new file mode 100644 index 000000000..fdf44d5ee --- /dev/null +++ b/rust/kernel/impl_flags.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bitflag type generator. + +/// Common helper for declaring bitflag and bitmask types. +/// +/// This macro takes as input: +/// - A struct declaration representing a bitmask type +/// (e.g., `pub struct Permissions(u32)`). +/// - An enumeration declaration representing individual bit flags +/// (e.g., `pub enum Permission { ... }`). +/// +/// And generates: +/// - The struct and enum types with appropriate `#[repr]` attributes. +/// - Implementations of common bitflag operators +/// ([`::core::ops::BitOr`], [`::core::ops::BitAnd`], etc.). +/// - Utility methods such as `.contains()` to check flags. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// bits::bit_u32, +/// impl_flags, // +/// }; +/// +/// impl_flags!( +/// /// Represents multiple permissions. +/// #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] +/// pub struct Permissions(u32); +/// +/// /// Represents a single permission. +/// #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// pub enum Permission { +/// /// Read permission. +/// Read = bit_u32(0), +/// +/// /// Write permission. +/// Write = bit_u32(1), +/// +/// /// Execute permission. +/// Execute = bit_u32(2), +/// } +/// ); +/// +/// // Combine multiple permissions using the bitwise OR (`|`) operator. +/// let mut read_write: Permissions = Permission::Read | Permission::Write; +/// assert!(read_write.contains(Permission::Read)); +/// assert!(read_write.contains(Permission::Write)); +/// assert!(!read_write.contains(Permission::Execute)); +/// assert!(read_write.contains_any(Permission::Read | Permission::Execute)); +/// assert!(read_write.contains_all(Permission::Read | Permission::Write)); +/// +/// // Using the bitwise OR assignment (`|=`) operator. +/// read_write |= Permission::Execute; +/// assert!(read_write.contains(Permission::Execute)); +/// +/// // Masking a permission with the bitwise AND (`&`) operator. +/// let read_only: Permissions = read_write & Permission::Read; +/// assert!(read_only.contains(Permission::Read)); +/// assert!(!read_only.contains(Permission::Write)); +/// +/// // Toggling permissions with the bitwise XOR (`^`) operator. +/// let toggled: Permissions = read_only ^ Permission::Read; +/// assert!(!toggled.contains(Permission::Read)); +/// +/// // Inverting permissions with the bitwise NOT (`!`) operator. +/// let negated = !read_only; +/// assert!(negated.contains(Permission::Write)); +/// assert!(!negated.contains(Permission::Read)); +/// ``` +#[macro_export] +macro_rules! impl_flags { + ( + $(#[$outer_flags:meta])* + $vis_flags:vis struct $flags:ident($ty:ty); + + $(#[$outer_flag:meta])* + $vis_flag:vis enum $flag:ident { + $( + $(#[$inner_flag:meta])* + $name:ident = $value:expr + ),+ $( , )? + } + ) => { + $(#[$outer_flags])* + #[repr(transparent)] + $vis_flags struct $flags($ty); + + $(#[$outer_flag])* + #[repr($ty)] + $vis_flag enum $flag { + $( + $(#[$inner_flag])* + $name = $value + ),+ + } + + impl ::core::convert::From<$flag> for $flags { + #[inline] + fn from(value: $flag) -> Self { + Self(value as $ty) + } + } + + impl ::core::convert::From<$flags> for $ty { + #[inline] + fn from(value: $flags) -> Self { + value.0 + } + } + + impl ::core::ops::BitOr for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl ::core::ops::BitOrAssign for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitOr<$flag> for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: $flag) -> Self::Output { + self | Self::from(rhs) + } + } + + impl ::core::ops::BitOrAssign<$flag> for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: $flag) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitAnd for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl ::core::ops::BitAndAssign for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitAnd<$flag> for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: $flag) -> Self::Output { + self & Self::from(rhs) + } + } + + impl ::core::ops::BitAndAssign<$flag> for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: $flag) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitXor for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + Self((self.0 ^ rhs.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitXorAssign for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::BitXor<$flag> for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: $flag) -> Self::Output { + self ^ Self::from(rhs) + } + } + + impl ::core::ops::BitXorAssign<$flag> for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: $flag) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::Not for $flags { + type Output = Self; + #[inline] + fn not(self) -> Self::Output { + Self((!self.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitOr for $flag { + type Output = $flags; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + $flags(self as $ty | rhs as $ty) + } + } + + impl ::core::ops::BitAnd for $flag { + type Output = $flags; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + $flags(self as $ty & rhs as $ty) + } + } + + impl ::core::ops::BitXor for $flag { + type Output = $flags; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + $flags((self as $ty ^ rhs as $ty) & $flags::all_bits()) + } + } + + impl ::core::ops::Not for $flag { + type Output = $flags; + #[inline] + fn not(self) -> Self::Output { + $flags((!(self as $ty)) & $flags::all_bits()) + } + } + + impl $flags { + /// Returns an empty instance where no flags are set. + #[inline] + pub const fn empty() -> Self { + Self(0) + } + + /// Returns a mask containing all valid flag bits. + #[inline] + pub const fn all_bits() -> $ty { + 0 $( | $value )+ + } + + /// Checks if a specific flag is set. + #[inline] + pub fn contains(self, flag: $flag) -> bool { + (self.0 & flag as $ty) == flag as $ty + } + + /// Checks if at least one of the provided flags is set. + #[inline] + pub fn contains_any(self, flags: $flags) -> bool { + (self.0 & flags.0) != 0 + } + + /// Checks if all of the provided flags are set. + #[inline] + pub fn contains_all(self, flags: $flags) -> bool { + (self.0 & flags.0) == flags.0 + } + } + }; +} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs new file mode 100644 index 000000000..1fdc3963e --- /dev/null +++ b/rust/kernel/init.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to the [`pin-init`] crate. +//! +//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential +//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! The [`pin-init`] crate is the way such structs are initialized on the Rust side. Please refer +//! to its documentation to better understand how to use it. Additionally, there are many examples +//! throughout the kernel, such as the types from the [`sync`] module. And the ones presented +//! below. +//! +//! [`sync`]: crate::sync +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [`pin-init`]: https://rust.docs.kernel.org/pin_init/ +//! +//! # [`Opaque`] +//! +//! For the special case where initializing a field is a single FFI-function call that cannot fail, +//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single +//! [`Opaque`] field by just delegating to the supplied closure. You can use these in +//! combination with [`pin_init!`]. +//! +//! [`Opaque`]: crate::types::Opaque +//! [`Opaque::ffi_init`]: crate::types::Opaque::ffi_init +//! [`pin_init!`]: pin_init::pin_init +//! +//! # Examples +//! +//! ## General Examples +//! +//! ```rust +//! # #![expect(clippy::undocumented_unsafe_blocks)] +//! use kernel::types::Opaque; +//! use pin_init::pin_init_from_closure; +//! +//! // assume we have some `raw_foo` type in C: +//! #[repr(C)] +//! struct RawFoo([u8; 16]); +//! extern "C" { +//! fn init_foo(_: *mut RawFoo); +//! } +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! raw: Opaque, +//! } +//! +//! impl Foo { +//! fn setup(self: Pin<&mut Self>) { +//! pr_info!("Setting up foo\n"); +//! } +//! } +//! +//! let foo = pin_init!(Foo { +//! raw <- unsafe { +//! Opaque::ffi_init(|s| { +//! // note that this cannot fail. +//! init_foo(s); +//! }) +//! }, +//! }).pin_chain(|foo| { +//! foo.setup(); +//! Ok(()) +//! }); +//! ``` +//! +//! ```rust +//! use kernel::{prelude::*, types::Opaque}; +//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; +//! # mod bindings { +//! # #![expect(non_camel_case_types, clippy::missing_safety_doc)] +//! # pub struct foo; +//! # pub unsafe fn init_foo(_ptr: *mut foo) {} +//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} +//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } +//! # } +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! foo: Opaque, +//! #[pin] +//! _p: PhantomPinned, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init::pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = addr_of_mut!((*slot).foo); +//! +//! // Initialize the `foo` +//! bindings::init_foo(Opaque::cast_into(foo)); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(Opaque::cast_into(foo), flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(Opaque::cast_into(foo)); +//! return Err(Error::from_errno(err)); +//! } +//! +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get()) }; +//! } +//! } +//! ``` + +use crate::{ + alloc::{AllocError, Flags}, + error::{self, Error}, +}; +use pin_init::{init_from_closure, pin_init_from_closure, Init, PinInit}; + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit: Sized { + /// Pinned version of `Self`. + /// + /// If a type already implicitly pins its pointee, `Pin` is unnecessary. In this case use + /// `Self`, otherwise just use `Pin`. + type PinnedSelf; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] + fn pin_init(init: impl PinInit, flags: Flags) -> error::Result + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| { + pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)) + }) + }; + Self::try_pin_init(init, flags) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From; + + /// Use the given initializer to in-place initialize a `T`. + #[inline] + fn init(init: impl Init, flags: Flags) -> error::Result + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))) + }; + Self::try_init(init, flags) + } +} + +/// Construct an in-place fallible initializer for `struct`s. +/// +/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use +/// [`init!`]. +/// +/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, +/// append `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::error::Error; +/// use pin_init::init_zeroed; +/// struct BigBuf { +/// big: KBox<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init { +/// try_init!(Self { +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, +/// small: [0; 1024 * 1024], +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Infallible`]: core::convert::Infallible +/// [`init!`]: pin_init::init +/// [`try_pin_init!`]: crate::try_pin_init! +/// [`Error`]: crate::error::Error +#[macro_export] +macro_rules! try_init { + ($($args:tt)*) => { + ::pin_init::init!( + #[default_error($crate::error::Error)] + $($args)* + ) + } +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. +/// +/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop +/// initialization and return the error. +/// +/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when +/// initialization fails, the memory can be safely deallocated without any further modifications. +/// +/// This macro defaults the error to [`Error`]. +/// +/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` +/// after the `struct` initializer to specify the error type you want to use. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(new_uninit)] +/// use kernel::error::Error; +/// use pin_init::init_zeroed; +/// #[pin_data] +/// struct BigBuf { +/// big: KBox<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// ptr: *mut u8, +/// } +/// +/// impl BigBuf { +/// fn new() -> impl PinInit { +/// try_pin_init!(Self { +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, +/// small: [0; 1024 * 1024], +/// ptr: core::ptr::null_mut(), +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Infallible`]: core::convert::Infallible +/// [`pin_init!`]: pin_init::pin_init +/// [`Error`]: crate::error::Error +#[macro_export] +macro_rules! try_pin_init { + ($($args:tt)*) => { + ::pin_init::pin_init!( + #[default_error($crate::error::Error)] + $($args)* + ) + } +} diff --git a/rust/kernel/interop.rs b/rust/kernel/interop.rs new file mode 100644 index 000000000..3b371d782 --- /dev/null +++ b/rust/kernel/interop.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for interfacing Rust code with C kernel subsystems. +//! +//! This module is intended for low-level, unsafe Rust infrastructure code +//! that interoperates between Rust and C. It is *not* for use directly in +//! Rust drivers. + +pub mod list; diff --git a/rust/kernel/interop/list.rs b/rust/kernel/interop/list.rs new file mode 100644 index 000000000..54265ea03 --- /dev/null +++ b/rust/kernel/interop/list.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust interface for C doubly circular intrusive linked lists. +//! +//! This module provides Rust abstractions for iterating over C `list_head`-based +//! linked lists. It should only be used for cases where C and Rust code share +//! direct access to the same linked list through a C interop interface. +//! +//! Note: This *must not* be used by Rust components that just need a linked list +//! primitive. Use [`kernel::list::List`] instead. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! bindings, +//! interop::list::clist_create, +//! types::Opaque, +//! }; +//! # // Create test list with values (0, 10, 20) - normally done by C code but it is +//! # // emulated here for doctests using the C bindings. +//! # use core::mem::MaybeUninit; +//! # +//! # /// C struct with embedded `list_head` (typically will be allocated by C code). +//! # #[repr(C)] +//! # pub struct SampleItemC { +//! # pub value: i32, +//! # pub link: bindings::list_head, +//! # } +//! # +//! # let mut head = MaybeUninit::::uninit(); +//! # +//! # let head = head.as_mut_ptr(); +//! # // SAFETY: `head` and all the items are test objects allocated in this scope. +//! # unsafe { bindings::INIT_LIST_HEAD(head) }; +//! # +//! # let mut items = [ +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # ]; +//! # +//! # for (i, item) in items.iter_mut().enumerate() { +//! # let ptr = item.as_mut_ptr(); +//! # // SAFETY: `ptr` points to a valid `MaybeUninit`. +//! # unsafe { (*ptr).value = i as i32 * 10 }; +//! # // SAFETY: `&raw mut` creates a pointer valid for `INIT_LIST_HEAD`. +//! # unsafe { bindings::INIT_LIST_HEAD(&raw mut (*ptr).link) }; +//! # // SAFETY: `link` was just initialized and `head` is a valid list head. +//! # unsafe { bindings::list_add_tail(&mut (*ptr).link, head) }; +//! # } +//! +//! /// Rust wrapper for the C struct. +//! /// +//! /// The list item struct in this example is defined in C code as: +//! /// +//! /// ```c +//! /// struct SampleItemC { +//! /// int value; +//! /// struct list_head link; +//! /// }; +//! /// ``` +//! #[repr(transparent)] +//! pub struct Item(Opaque); +//! +//! impl Item { +//! pub fn value(&self) -> i32 { +//! // SAFETY: `Item` has the same layout as `SampleItemC`. +//! unsafe { (*self.0.get()).value } +//! } +//! } +//! +//! // Create typed [`CList`] from sentinel head. +//! // SAFETY: `head` is valid and initialized, items are `SampleItemC` with +//! // embedded `link` field, and `Item` is `#[repr(transparent)]` over `SampleItemC`. +//! let list = unsafe { clist_create!(head, Item, SampleItemC, link) }; +//! +//! // Iterate directly over typed items. +//! let mut found_0 = false; +//! let mut found_10 = false; +//! let mut found_20 = false; +//! +//! for item in list.iter() { +//! let val = item.value(); +//! if val == 0 { found_0 = true; } +//! if val == 10 { found_10 = true; } +//! if val == 20 { found_20 = true; } +//! } +//! +//! assert!(found_0 && found_10 && found_20); +//! ``` + +use core::{ + iter::FusedIterator, + marker::PhantomData, // +}; + +use crate::{ + bindings, + types::Opaque, // +}; + +use pin_init::{ + pin_data, + pin_init, + PinInit, // +}; + +/// FFI wrapper for a C `list_head` object used in intrusive linked lists. +/// +/// # Invariants +/// +/// - The underlying `list_head` is initialized with valid non-`NULL` `next`/`prev` pointers. +#[pin_data] +#[repr(transparent)] +pub struct CListHead { + #[pin] + inner: Opaque, +} + +impl CListHead { + /// Create a `&CListHead` reference from a raw `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked `list_head` nodes must not be modified from + /// anywhere for the lifetime `'a`, unless done so via any [`CListHead`] APIs. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CListHead` has the same layout as `list_head`. + // - `ptr` is valid and unmodified for `'a` per caller guarantees. + unsafe { &*ptr.cast() } + } + + /// Get the raw `list_head` pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::list_head { + self.inner.get() + } + + /// Get the next [`CListHead`] in the list. + #[inline] + pub fn next(&self) -> &Self { + let raw = self.as_raw(); + // SAFETY: + // - `self.as_raw()` is valid and initialized per type invariants. + // - The `next` pointer is valid and non-`NULL` per type invariants + // (initialized via `INIT_LIST_HEAD()` or equivalent). + unsafe { Self::from_raw((*raw).next) } + } + + /// Check if this node is linked in a list (not isolated). + #[inline] + pub fn is_linked(&self) -> bool { + let raw = self.as_raw(); + // SAFETY: `self.as_raw()` is valid per type invariants. + unsafe { (*raw).next != raw && (*raw).prev != raw } + } + + /// Returns a pin-initializer for the list head. + pub fn new() -> impl PinInit { + pin_init!(Self { + // SAFETY: `INIT_LIST_HEAD` initializes `slot` to a valid empty list. + inner <- Opaque::ffi_init(|slot| unsafe { bindings::INIT_LIST_HEAD(slot) }), + }) + } +} + +// SAFETY: `list_head` contains no thread-bound state; it only holds +// `next`/`prev` pointers. +unsafe impl Send for CListHead {} + +// SAFETY: `CListHead` can be shared among threads as modifications are +// not allowed at the moment. +unsafe impl Sync for CListHead {} + +impl PartialEq for CListHead { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl Eq for CListHead {} + +/// Low-level iterator over `list_head` nodes. +/// +/// An iterator used to iterate over a C intrusive linked list (`list_head`). The caller has to +/// perform conversion of returned [`CListHead`] to an item (using [`container_of`] or similar). +/// +/// # Invariants +/// +/// `current` and `sentinel` are valid references into an initialized linked list. +struct CListHeadIter<'a> { + /// Current position in the list. + current: &'a CListHead, + /// The sentinel head (used to detect end of iteration). + sentinel: &'a CListHead, +} + +impl<'a> Iterator for CListHeadIter<'a> { + type Item = &'a CListHead; + + #[inline] + fn next(&mut self) -> Option { + // Check if we've reached the sentinel (end of list). + if self.current == self.sentinel { + return None; + } + + let item = self.current; + self.current = item.next(); + Some(item) + } +} + +impl<'a> FusedIterator for CListHeadIter<'a> {} + +/// A typed C linked list with a sentinel head intended for FFI use-cases where +/// a C subsystem manages a linked list that Rust code needs to read. Generally +/// required only for special cases. +/// +/// A sentinel head [`CListHead`] represents the entire linked list and can be used +/// for iteration over items of type `T`; it is not associated with a specific item. +/// +/// The const generic `OFFSET` specifies the byte offset of the `list_head` field within +/// the struct that `T` wraps. +/// +/// # Invariants +/// +/// - The sentinel [`CListHead`] has valid non-`NULL` `next`/`prev` pointers. +/// - `OFFSET` is the byte offset of the `list_head` field within the struct that `T` wraps. +/// - All the list's `list_head` nodes have valid non-`NULL` `next`/`prev` pointers. +#[repr(transparent)] +pub struct CList(CListHead, PhantomData); + +impl CList { + /// Create a typed [`CList`] reference from a raw sentinel `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized sentinel `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked nodes must not be concurrently modified for the lifetime `'a`. + /// - The list must contain items where the `list_head` field is at byte offset `OFFSET`. + /// - `T` must be `#[repr(transparent)]` over the C struct. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CList` has the same layout as `CListHead` due to `#[repr(transparent)]`. + // - Caller guarantees `ptr` is a valid, sentinel `list_head` object. + unsafe { &*ptr.cast() } + } + + /// Check if the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + !self.0.is_linked() + } + + /// Create an iterator over typed items. + #[inline] + pub fn iter(&self) -> CListIter<'_, T, OFFSET> { + let head = &self.0; + CListIter { + head_iter: CListHeadIter { + current: head.next(), + sentinel: head, + }, + _phantom: PhantomData, + } + } +} + +/// High-level iterator over typed list items. +pub struct CListIter<'a, T, const OFFSET: usize> { + head_iter: CListHeadIter<'a>, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T, const OFFSET: usize> Iterator for CListIter<'a, T, OFFSET> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option { + let head = self.head_iter.next()?; + + // Convert to item using `OFFSET`. + // + // SAFETY: The pointer calculation is valid because `OFFSET` is derived + // from `offset_of!` per type invariants. + Some(unsafe { &*head.as_raw().byte_sub(OFFSET).cast::() }) + } +} + +impl<'a, T, const OFFSET: usize> FusedIterator for CListIter<'a, T, OFFSET> {} + +/// Create a C doubly-circular linked list interface [`CList`] from a raw `list_head` pointer. +/// +/// This macro creates a `CList` that can iterate over items of type `$rust_type` +/// linked via the `$field` field in the underlying C struct `$c_type`. +/// +/// # Arguments +/// +/// - `$head`: Raw pointer to the sentinel `list_head` object (`*mut bindings::list_head`). +/// - `$rust_type`: Each item's Rust wrapper type. +/// - `$c_type`: Each item's C struct type that contains the embedded `list_head`. +/// - `$field`: The name of the `list_head` field within the C struct. +/// +/// # Safety +/// +/// The caller must ensure: +/// +/// - `$head` is a valid, initialized sentinel `list_head` (e.g. via `INIT_LIST_HEAD()`) +/// pointing to a list that is not concurrently modified for the lifetime of the [`CList`]. +/// - The list contains items of type `$c_type` linked via an embedded `$field`. +/// - `$rust_type` is `#[repr(transparent)]` over `$c_type` or has compatible layout. +/// +/// # Examples +/// +/// Refer to the examples in the [`crate::interop::list`] module documentation. +#[macro_export] +macro_rules! clist_create { + ($head:expr, $rust_type:ty, $c_type:ty, $($field:tt).+) => {{ + // Compile-time check that field path is a `list_head`. + let _: fn(*const $c_type) -> *const $crate::bindings::list_head = + |p| &raw const (*p).$($field).+; + + // Calculate offset and create `CList`. + const OFFSET: usize = ::core::mem::offset_of!($c_type, $($field).+); + $crate::interop::list::CList::<$rust_type, OFFSET>::from_raw($head) + }}; +} +pub use clist_create; diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000..a880ec3b8 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local processor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # ⚠️ Warning! ⚠️ +//! +//! The usage of this module can be more complicated than meets the eye, especially surrounding +//! [preemptible kernels]. It's recommended to take care when using the functions and types defined +//! here and familiarize yourself with the various documentation we have before using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preemptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible kernel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local +/// processor interrupt state management on their own and managing this by hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this + // object, hence calling the corresponding `local_interrupt_enable()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible kernels. + /// + /// This can be used for annotating code that is known to be run in contexts where local + /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs new file mode 100644 index 000000000..5ce9fd129 --- /dev/null +++ b/rust/kernel/io.rs @@ -0,0 +1,1754 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) + +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + +use crate::{ + bindings, + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // +}; + +#[cfg(CONFIG_HAS_IOMEM)] +pub mod mem; +pub mod poll; +pub mod register; +pub mod resource; + +pub use crate::register; +pub use resource::Resource; + +use register::LocatedRegister; + +/// Physical address type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type PhysAddr = bindings::phys_addr_t; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::resource_size_t; + +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region { + inner: [u8], +} + +impl Region { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl KnownSize for Region { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl IntoBytes for Region { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + +/// Raw representation of an MMIO region. +/// +/// `MmioRaw` is equivalent to `T __iomem *` in C. +/// +/// By itself, the existence of an instance of this structure does not provide any guarantees that +/// the represented MMIO region does exist or is properly mapped. +/// +/// Instead, the bus specific MMIO implementation must convert this raw representation into an +/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` +/// structure any guarantees are given. +pub struct MmioRaw { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, +} + +impl Copy for MmioRaw {} +impl Clone for MmioRaw { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Send for MmioRaw {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Sync for MmioRaw {} + +impl MmioRaw { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), + } + } +} + +impl MmioRaw> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) + } +} + +impl MmioRaw { + /// Returns the base address of the MMIO region. + #[inline] + pub fn addr(&self) -> usize { + self.ptr.addr() + } + + /// Returns the size of the MMIO region. + #[inline] + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) + } +} + +/// Checks whether an access of type `U` at the given `base` and the given `offset` +/// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. +#[inline] +const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::()) { + end <= size && (base.wrapping_add(offset) % align_of::() == 0) + } else { + false + } +} + +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> ::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + +/// Trait indicating that an I/O backend supports operations of a certain type and providing an +/// implementation for these operations. +/// +/// Different I/O backends can implement this trait to expose only the operations they support. +/// +/// For example, a PCI configuration space may implement `IoCapable`, `IoCapable`, +/// and `IoCapable`, but not `IoCapable`, while an MMIO region on a 64-bit +/// system might implement all four. +pub trait IoCapable: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; + + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); +} + +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); + + /// Copy contents from `buffer` to `view`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + let mut buf = MaybeUninit::::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } +} + +/// Describes a given I/O location: its offset, width, and type to convert the raw value from and +/// into. +/// +/// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and +/// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and +/// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets +/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] +/// macro). +/// +/// An `IoLoc` carries the following pieces of information: +/// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. +/// - The offset to access (returned by [`IoLoc::offset`]), +/// - The width of the access (determined by [`IoLoc::IoType`]), +/// - The type `T` in which the raw data is returned or provided. +/// +/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type +/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +pub trait IoLoc { + /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). + type IoType: Into + From; + + /// Consumes `self` and returns the offset of this location. + fn offset(self) -> usize; +} + +/// Implements [`IoLoc, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. +macro_rules! impl_usize_ioloc { + ($($ty:ty),*) => { + $( + impl IoLoc, $ty> for usize { + type IoType = $ty; + + #[inline(always)] + fn offset(self) -> usize { + self + } + } + )* + } +} + +// Provide the ability to read any primitive type from a [`usize`]. +impl_usize_ioloc!(u8, u16, u32, u64); + +/// Types implementing this trait (e.g. MMIO BARs or PCI config regions) +/// can perform I/O operations on regions of memory. +/// +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. +/// +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> ::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. +/// +/// This trait provides: +/// - Helper methods for offset validation and address calculation +/// - Fallible (runtime checked) accessors for different data widths +/// +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } + + /// Returns the length of the slice in number of elements. + #[inline] + fn len(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } + + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + fn try_cast(self) -> Result<::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::() > KnownSize::size(ptr) { + return Err(EINVAL); + } + + if ptr.addr() % align_of::() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } + } + + /// Fallible 8-bit read with runtime bounds check. + #[inline(always)] + fn try_read8(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 16-bit read with runtime bounds check. + #[inline(always)] + fn try_read16(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 32-bit read with runtime bounds check. + #[inline(always)] + fn try_read32(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 64-bit read with runtime bounds check. + #[inline(always)] + fn try_read64(self, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_read(offset) + } + + /// Fallible 8-bit write with runtime bounds check. + #[inline(always)] + fn try_write8(self, value: u8, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 16-bit write with runtime bounds check. + #[inline(always)] + fn try_write16(self, value: u16, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 32-bit write with runtime bounds check. + #[inline(always)] + fn try_write32(self, value: u32, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Fallible 64-bit write with runtime bounds check. + #[inline(always)] + fn try_write64(self, value: u64, offset: usize) -> Result + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.try_write(offset, value) + } + + /// Infallible 8-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read8(self, offset: usize) -> u8 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 16-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read16(self, offset: usize) -> u16 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 32-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read32(self, offset: usize) -> u32 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 64-bit read with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn read64(self, offset: usize) -> u64 + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.read(offset) + } + + /// Infallible 8-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write8(self, value: u8, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 16-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write16(self, value: u16, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 32-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write32(self, value: u32, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Infallible 64-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write64(self, value: u64, offset: usize) + where + usize: IoLoc, + Self::Backend: IoCapable, + { + self.write(offset, value) + } + + /// Generic fallible read with runtime bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region>) -> Result { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.try_read(0x10)?; + /// + /// // 8-bit read from address `0xfff`. + /// let v: u8 = io.try_read(0xfff)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_read(self, location: L) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view::(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) + } + + /// Generic fallible write with runtime bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region>) -> Result { + /// // 32-bit write of value `1` at address `0x10`. + /// io.try_write(0x10, 1u32)?; + /// + /// // 8-bit write of value `0xff` at address `0xfff`. + /// io.try_write(0xfff, 0xffu8)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_write(self, location: L, value: T) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view::(self, location.offset())?; + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + Ok(()) + } + + /// Generic fallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { + /// + /// io.try_write_reg(VERSION::new(1, 0)) + /// } + /// ``` + #[inline(always)] + fn try_write_reg(self, value: V) -> Result + where + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, + { + let (location, value) = value.into_io_op(); + + self.try_write(location, value) + } + + /// Generic fallible update with runtime bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { + /// io.try_update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn try_update(self, location: L, f: F) -> Result + where + L: IoLoc, + Self::Backend: IoCapable, + F: FnOnce(T) -> T, + { + let view = io_view::(self, location.offset())?; + + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); + + Ok(()) + } + + /// Generic infallible read with compile-time bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.read(0x10); + /// + /// // 8-bit read from the top of the I/O space. + /// let v: u8 = io.read(0xfff); + /// } + /// ``` + #[inline(always)] + fn read(self, location: L) -> T + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view_assert::(self, location.offset()); + Self::Backend::io_read(view).into() + } + + /// Generic infallible write with compile-time bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit write of value `1` at address `0x10`. + /// io.write(0x10, 1u32); + /// + /// // 8-bit write of value `0xff` at the top of the I/O space. + /// io.write(0xfff, 0xffu8); + /// } + /// ``` + #[inline(always)] + fn write(self, location: L, value: T) + where + L: IoLoc, + Self::Backend: IoCapable, + { + let view = io_view_assert::(self, location.offset()); + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + } + + /// Generic infallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { + /// io.write_reg(VERSION::new(1, 0)); + /// } + /// ``` + #[inline(always)] + fn write_reg(self, value: V) + where + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, + { + let (location, value) = value.into_io_op(); + + self.write(location, value) + } + + /// Generic infallible update with compile-time bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { + /// io.update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn update(self, location: L, f: F) + where + L: IoLoc, + Self::Backend: IoCapable, + F: FnOnce(T) -> T, + { + let view = io_view_assert::(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); + } +} + +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + +/// A view of memory-mapped I/O region. +/// +/// # Invariant +/// +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for Mmio<'_, T> {} +impl Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for Mmio<'_, T> {} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_mmio_io_capable { + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } + } + + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } + } + } + }; +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); +// MMIO regions on 64-bit systems also support 64-bit accesses. +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); + +impl IoCopyable for MmioBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } + } +} + +/// [`Mmio`] but using relaxed accessors. +/// +/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of +/// the regular ones. +/// +/// See [`Mmio::relaxed`] for a usage example. +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); + +impl Copy for RelaxedMmio<'_, T> {} +impl Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. + /// + /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses + /// and can be used when such ordering is not required. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// RelaxedMmio, + /// }; + /// + /// fn do_io(io: Mmio<'_, Region<0x100>>) { + /// // The access is performed using `readl_relaxed` instead of `readl`. + /// let v = io.relaxed().read32(0x10); + /// } + /// + /// ``` + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) + } +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); +// MMIO regions on 64-bit systems also support 64-bit accesses. +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for SysMem<'_, T> {} +impl Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl Copy for IoSysMap<'_, T> {} +impl Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl IoCapable for IoSysMapBackend +where + MmioBackend: IoCapable, + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper(pub T); + +impl<'a, T> ProjectHelper +where + T: Io<'a, Backend: IoBackend = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view( + self, + ptr: *mut U, + ) -> ::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs new file mode 100644 index 000000000..32a919099 --- /dev/null +++ b/rust/kernel/io/mem.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic memory-mapped IO. + +use crate::{ + device::{ + Bound, + Device, // + }, + devres::DevresLt, + io::{ + self, + resource::{ + Region, + Resource, // + }, + IoBase, + Mmio, + MmioBackend, + MmioRaw, // + }, + prelude::*, + types::{ + CovariantForLt, + ForLt, // + }, +}; + +/// An IO request for a specific device and resource. +pub struct IoRequest<'a> { + device: &'a Device, + resource: &'a Resource, +} + +impl<'a> IoRequest<'a> { + /// Creates a new [`IoRequest`] instance. + /// + /// # Safety + /// + /// Callers must ensure that `resource` is valid for `device` during the + /// lifetime `'a`. + pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> Self { + IoRequest { device, resource } + } + + /// Maps an [`IoRequest`] where the size is known at compile time. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{ + /// bindings, + /// device::Core, + /// io::Io, + /// of, + /// platform, + /// }; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// # type Data<'bound> = Self; + /// + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { + /// let offset = 0; // Some offset. + /// + /// // If the size is known at compile time, use [`Self::iomap_sized`]. + /// // + /// // No runtime checks will apply when reading and writing. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap_sized::<42>()?; + /// + /// // Read and write a 32-bit value at `offset`. + /// let data = iomem.read32(offset); + /// + /// iomem.write32(data, offset); + /// + /// # Ok(SampleDriver) + /// } + /// } + /// ``` + pub fn iomap_sized(self) -> Result> { + IoMem::ioremap(self.device, self.resource) + } + + /// Same as [`Self::iomap_sized`] but with exclusive access to the + /// underlying region. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + pub fn iomap_exclusive_sized(self) -> Result> { + ExclusiveIoMem::ioremap(self.device, self.resource) + } + + /// Maps an [`IoRequest`] where the size is not known at compile time, + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{ + /// bindings, + /// device::Core, + /// io::Io, + /// of, + /// platform, + /// }; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// # type Data<'bound> = Self; + /// + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { + /// let offset = 0; // Some offset. + /// + /// // Unlike [`Self::iomap_sized`], here the size of the memory region + /// // is not known at compile time, so only the `try_read*` and `try_write*` + /// // family of functions should be used, leading to runtime checks on every + /// // access. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap()?; + /// + /// let data = iomem.try_read32(offset)?; + /// + /// iomem.try_write32(data, offset)?; + /// + /// # Ok(SampleDriver) + /// } + /// } + /// ``` + pub fn iomap(self) -> Result> { + self.iomap_sized::<0>() + } + + /// Same as [`Self::iomap`] but with exclusive access to the underlying + /// region. + pub fn iomap_exclusive(self) -> Result> { + self.iomap_exclusive_sized::<0>() + } +} + +/// An exclusive memory-mapped IO region. +/// +/// # Invariants +/// +/// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. +pub struct ExclusiveIoMem<'a, const SIZE: usize> { + /// The underlying `IoMem` instance. + iomem: IoMem<'a, SIZE>, + + /// The region abstraction. This represents exclusive access to the + /// range represented by the underlying `iomem`. + /// + /// This field is needed for ownership of the region. + _region: Region, +} + +impl ForLt for ExclusiveIoMem<'static, SIZE> { + type Of<'a> = ExclusiveIoMem<'a, SIZE>; +} + +// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`, +// which holds `&'a Device`, which is covariant. +unsafe impl CovariantForLt for ExclusiveIoMem<'static, SIZE> {} + +/// A device-managed exclusive I/O memory region. +/// +/// See [`ExclusiveIoMem::into_devres`]. +pub type DevresExclusiveIoMem = DevresLt>; + +impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { + /// Creates a new `ExclusiveIoMem` instance. + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { + let start = resource.start(); + let size = resource.size(); + let name = resource.name().unwrap_or_default(); + + let region = resource + .request_region( + start, + size, + name.to_cstring()?, + io::resource::Flags::IORESOURCE_MEM, + ) + .ok_or(EBUSY)?; + + let iomem = IoMem::ioremap(dev, resource)?; + + Ok(ExclusiveIoMem { + iomem, + _region: region, + }) + } + + /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in + /// driver data. Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.iomem.dev; + // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of + // which remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() + } +} + +/// A generic memory-mapped IO region. +/// +/// Accesses to the underlying region is checked either at compile time, if the +/// region's size is known at that point, or at runtime otherwise. +/// +/// # Invariants +/// +/// [`IoMem`] always holds an [`MmioRaw`] instance that holds a valid pointer to the +/// start of the I/O memory mapped region. +pub struct IoMem<'a, const SIZE: usize = 0> { + dev: &'a Device, + io: MmioRaw>, +} + +impl ForLt for IoMem<'static, SIZE> { + type Of<'a> = IoMem<'a, SIZE>; +} + +// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for IoMem<'static, SIZE> {} + +/// A device-managed I/O memory region. +/// +/// See [`IoMem::into_devres`]. +pub type DevresIoMem = DevresLt>; + +impl<'a, const SIZE: usize> IoMem<'a, SIZE> { + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { + // Note: Some ioremap() implementations use types that depend on the CPU + // word width rather than the bus address width. + // + // TODO: Properly address this in the C code to avoid this `try_into`. + let size = resource.size().try_into()?; + if size == 0 { + return Err(EINVAL); + } + + let res_start = resource.start(); + + let addr = if resource + .flags() + .contains(io::resource::Flags::IORESOURCE_MEM_NONPOSTED) + { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap_np(res_start, size) } + } else { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap(res_start, size) } + }; + + if addr.is_null() { + return Err(ENOMEM); + } + + let io = MmioRaw::new_region(addr as usize, size)?; + Ok(IoMem { dev, io }) + } + + /// Consume the `IoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data. + /// Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.dev; + // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } + } +} + +impl Drop for IoMem<'_, SIZE> { + fn drop(&mut self) { + // SAFETY: Safe as by the invariant of `Io`. + unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + // SAFETY: Safe as by the invariant of `IoMem`. + unsafe { Mmio::from_raw(self.io) } + } +} diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs new file mode 100644 index 000000000..d75f2fcf4 --- /dev/null +++ b/rust/kernel/io/poll.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IO polling. +//! +//! C header: [`include/linux/iopoll.h`](srctree/include/linux/iopoll.h). + +use crate::{ + prelude::*, + processor::cpu_relax, + task::might_sleep, + time::{ + delay::{ + fsleep, + udelay, // + }, + Delta, + Instant, + Monotonic, // + }, +}; + +/// Polls periodically until a condition is met, an error occurs, +/// or the timeout is reached. +/// +/// The function repeatedly executes the given operation `op` closure and +/// checks its result using the condition closure `cond`. +/// +/// If `cond` returns `true`, the function returns successfully with +/// the result of `op`. Otherwise, it waits for a duration specified +/// by `sleep_delta` before executing `op` again. +/// +/// This process continues until either `op` returns an error, `cond` +/// returns `true`, or the timeout specified by `timeout_delta` is +/// reached. +/// +/// This function can only be used in a nonatomic context. +/// +/// # Errors +/// +/// If `op` returns an error, then that error is returned directly. +/// +/// If the timeout specified by `timeout_delta` is reached, then +/// `Err(ETIMEDOUT)` is returned. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::io::{ +/// Io, +/// Mmio, +/// Region, +/// poll::read_poll_timeout, // +/// }; +/// use kernel::time::Delta; +/// +/// const HW_READY: u16 = 0x01; +/// +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { +/// read_poll_timeout( +/// // The `op` closure reads the value of a specific status register. +/// || io.try_read16(0x1000), +/// // The `cond` closure takes a reference to the value returned by `op` +/// // and checks whether the hardware is ready. +/// |val: &u16| *val == HW_READY, +/// Delta::from_millis(50), +/// Delta::from_secs(3), +/// )?; +/// Ok(()) +/// } +/// ``` +#[track_caller] +pub fn read_poll_timeout( + mut op: Op, + mut cond: Cond, + sleep_delta: Delta, + timeout_delta: Delta, +) -> Result +where + Op: FnMut() -> Result, + Cond: FnMut(&T) -> bool, +{ + let start: Instant = Instant::now(); + + // Unlike the C version, we always call `might_sleep()` unconditionally, + // as conditional calls are error-prone. We clearly separate + // `read_poll_timeout()` and `read_poll_timeout_atomic()` to aid + // tools like klint. + might_sleep(); + + loop { + let val = op()?; + if cond(&val) { + // Unlike the C version, we immediately return. + // We know the condition is met so we don't need to check again. + return Ok(val); + } + + if start.elapsed() > timeout_delta { + // Unlike the C version, we immediately return. + // We have just called `op()` so we don't need to call it again. + return Err(ETIMEDOUT); + } + + if !sleep_delta.is_zero() { + fsleep(sleep_delta); + } + + // `fsleep()` could be a busy-wait loop so we always call `cpu_relax()`. + cpu_relax(); + } +} + +/// Polls periodically until a condition is met, an error occurs, +/// or the attempt limit is reached. +/// +/// The function repeatedly executes the given operation `op` closure and +/// checks its result using the condition closure `cond`. +/// +/// If `cond` returns `true`, the function returns successfully with the result of `op`. +/// Otherwise, it performs a busy wait for a duration specified by `delay_delta` +/// before executing `op` again. +/// +/// This process continues until either `op` returns an error, `cond` +/// returns `true`, or the attempt limit specified by `retry` is reached. +/// +/// # Errors +/// +/// If `op` returns an error, then that error is returned directly. +/// +/// If the attempt limit specified by `retry` is reached, then +/// `Err(ETIMEDOUT)` is returned. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::io::{ +/// Io, +/// Mmio, +/// Region, +/// poll::read_poll_timeout_atomic, // +/// }; +/// use kernel::time::Delta; +/// +/// const HW_READY: u16 = 0x01; +/// +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { +/// read_poll_timeout_atomic( +/// // The `op` closure reads the value of a specific status register. +/// || io.try_read16(0x1000), +/// // The `cond` closure takes a reference to the value returned by `op` +/// // and checks whether the hardware is ready. +/// |val: &u16| *val == HW_READY, +/// Delta::from_micros(50), +/// 1000, +/// )?; +/// Ok(()) +/// } +/// ``` +pub fn read_poll_timeout_atomic( + mut op: Op, + mut cond: Cond, + delay_delta: Delta, + retry: usize, +) -> Result +where + Op: FnMut() -> Result, + Cond: FnMut(&T) -> bool, +{ + for _ in 0..retry { + let val = op()?; + if cond(&val) { + return Ok(val); + } + + if !delay_delta.is_zero() { + udelay(delay_delta); + } + + cpu_relax(); + } + + Err(ETIMEDOUT) +} diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs new file mode 100644 index 000000000..03dfd2ff4 --- /dev/null +++ b/rust/kernel/io/register.rs @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Macro to define register layout and accessors. +//! +//! The [`register!`](kernel::io::register!) macro provides an intuitive and readable syntax for +//! defining a dedicated type for each register and accessing it using [`Io`](super::Io). Each such +//! type comes with its own field accessors that can return an error if a field's value is invalid. +//! +//! Note: most of the items in this module are public so they can be referenced by the macro, but +//! most are not to be used directly by users. Outside of the `register!` macro itself, the only +//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! +//! # Simple example +//! +//! ```no_run +//! use kernel::io::register; +//! +//! register! { +//! /// Basic information about the chip. +//! pub BOOT_0(u32) @ 0x00000100 { +//! /// Vendor ID. +//! 15:8 vendor_id; +//! /// Major revision of the chip. +//! 7:4 major_revision; +//! /// Minor revision of the chip. +//! 3:0 minor_revision; +//! } +//! } +//! ``` +//! +//! This defines a 32-bit `BOOT_0` type which can be read from or written to offset `0x100` of an +//! `Io` region, with the described bitfields. For instance, `minor_revision` consists of the 4 +//! least significant bits of the type. +//! +//! Fields are instances of [`Bounded`](kernel::num::Bounded) and can be read by calling their +//! getter method, which is named after them. They also have setter methods prefixed with `with_` +//! for runtime values and `with_const_` for constant values. All setters return the updated +//! register value. +//! +//! Fields can also be transparently converted from/to an arbitrary type by using the `=>` and +//! `?=>` syntaxes. +//! +//! If present, doc comments above register or fields definitions are added to the relevant item +//! they document (the register type itself, or the field's setter and getter methods). +//! +//! Note that multiple registers can be defined in a single `register!` invocation. This can be +//! useful to group related registers together. +//! +//! Here is how the register defined above can be used in code: +//! +//! +//! ```no_run +//! use kernel::{ +//! io::{ +//! register, +//! Io, +//! IoLoc, +//! }, +//! num::Bounded, +//! }; +//! # use kernel::io::{Mmio, Region}; +//! # register! { +//! # pub BOOT_0(u32) @ 0x00000100 { +//! # 15:8 vendor_id; +//! # 7:4 major_revision; +//! # 3:0 minor_revision; +//! # } +//! # } +//! # fn test(io: Mmio<'_, Region<0x1000>>) { +//! # fn obtain_vendor_id() -> u8 { 0xff } +//! +//! // Read from the register's defined offset (0x100). +//! let boot0 = io.read(BOOT_0); +//! pr_info!("chip revision: {}.{}", boot0.major_revision().get(), boot0.minor_revision().get()); +//! +//! // Update some fields and write the new value back. +//! let new_boot0 = boot0 +//! // Constant values. +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! // Runtime value. +//! .with_vendor_id(obtain_vendor_id()); +//! io.write_reg(new_boot0); +//! +//! // Or, build a new value from zero and write it: +//! io.write_reg(BOOT_0::zeroed() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Or, read and update the register in a single step. +//! io.update(BOOT_0, |r| r +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Constant values can also be built using the const setters. +//! const V: BOOT_0 = pin_init::zeroed::() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>(); +//! # } +//! ``` +//! +//! For more extensive documentation about how to define registers, see the +//! [`register!`](kernel::io::register!) macro. + +use core::marker::PhantomData; + +use crate::{ + build_assert::build_assert, + io::IoLoc, // +}; + +use super::Region; + +/// Trait implemented by all registers. +pub trait Register: Sized { + /// Backing primitive type of the register. + type Storage: Into + From; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; +} + +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Register {} + +/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when +/// passing a [`FixedRegister`] value. +impl IoLoc, T> for () +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used +/// as an [`IoLoc`]. +impl IoLoc, T> for T +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Location of a fixed register. +pub struct FixedRegisterLoc(PhantomData); + +impl FixedRegisterLoc { + /// Returns the location of `T`. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl IoLoc, T> for FixedRegisterLoc +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Trait providing a base address to be added to the offset of a relative register to obtain +/// its actual offset. +/// +/// The `T` generic argument is used to distinguish which base to use, in case a type provides +/// several bases. It is given to the `register!` macro to restrict the use of the register to +/// implementors of this particular variant. +pub trait RegisterBase { + /// Base address to which register offsets are added. + const BASE: usize; +} + +/// Trait implemented by all registers that are relative to a base. +pub trait WithBase { + /// Family of bases applicable to this register. + type BaseFamily; + + /// Returns the absolute location of this type when using `B` as its base. + #[inline(always)] + fn of>() -> RelativeRegisterLoc + where + Self: Register, + { + RelativeRegisterLoc::new() + } +} + +/// Trait implemented by relative registers. +pub trait RelativeRegister: Register + WithBase {} + +/// Location of a relative register. +/// +/// This can either be an immediately accessible regular [`RelativeRegister`], or a +/// [`RelativeRegisterArray`] that needs one additional resolution through +/// [`RelativeRegisterLoc::at`]. +pub struct RelativeRegisterLoc(PhantomData, PhantomData); + +impl RelativeRegisterLoc +where + T: Register + WithBase, + B: RegisterBase + ?Sized, +{ + /// Returns the location of a relative register or register array. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData, PhantomData) + } + + // Returns the absolute offset of the relative register using base `B`. + // + // This is implemented as a private const method so it can be reused by the [`IoLoc`] + // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. + #[inline] + const fn offset(self) -> usize { + B::BASE + T::OFFSET + } +} + +impl IoLoc, T> for RelativeRegisterLoc +where + T: RelativeRegister, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + RelativeRegisterLoc::offset(self) + } +} + +/// Trait implemented by arrays of registers. +pub trait RegisterArray: Register { + /// Number of elements in the registers array. + const SIZE: usize; + /// Number of bytes between the start of elements in the registers array. + const STRIDE: usize; +} + +/// Location of an array register. +pub struct RegisterArrayLoc(usize, PhantomData); + +impl RegisterArrayLoc { + /// Returns the location of register `T` at position `idx`, with build-time validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(idx, PhantomData) + } + + /// Attempts to return the location of register `T` at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(idx, PhantomData)) + } else { + None + } + } +} + +impl IoLoc, T> for RegisterArrayLoc +where + T: RegisterArray, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + self.0 * T::STRIDE + } +} + +/// Trait providing location builders for [`RegisterArray`]s. +pub trait Array { + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + fn at(idx: usize) -> RegisterArrayLoc + where + Self: RegisterArray, + { + RegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + fn try_at(idx: usize) -> Option> + where + Self: RegisterArray, + { + RegisterArrayLoc::try_new(idx) + } +} + +/// Trait implemented by arrays of relative registers. +pub trait RelativeRegisterArray: RegisterArray + WithBase {} + +/// Location of a relative array register. +pub struct RelativeRegisterArrayLoc< + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +>(RelativeRegisterLoc, usize); + +impl RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of register `T` from the base `B` at index `idx`, with build-time + /// validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(RelativeRegisterLoc::new(), idx) + } + + /// Attempts to return the location of register `T` from the base `B` at index `idx`, with + /// runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(RelativeRegisterLoc::new(), idx)) + } else { + None + } + } +} + +/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. +impl RelativeRegisterLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc { + RelativeRegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_at(self, idx: usize) -> Option> { + RelativeRegisterArrayLoc::try_new(idx) + } +} + +impl IoLoc, T> for RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + self.0.offset() + self.1 * T::STRIDE + } +} + +/// Trait implemented by items that contain both a register value and the absolute I/O location at +/// which to write it. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait LocatedRegister { + /// Register value to write. + type Value: Register; + /// Full location information at which to write the value. + type Location: IoLoc; + + /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write + /// operation. + fn into_io_op(self) -> (Self::Location, Self::Value); +} + +impl LocatedRegister> for T +where + T: FixedRegister, +{ + type Location = FixedRegisterLoc; + type Value = T; + + #[inline(always)] + fn into_io_op(self) -> (FixedRegisterLoc, T) { + (FixedRegisterLoc::new(), self) + } +} + +/// Defines a dedicated type for a register, including getter and setter methods for its fields and +/// methods to read and write it from an [`Io`](kernel::io::Io) region. +/// +/// This documentation focuses on how to declare registers. See the [module-level +/// documentation](mod@kernel::io::register) for examples of how to access them. +/// +/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of +/// registers, and relative arrays of registers. +/// +/// ## Fixed offset registers +/// +/// These are the simplest kind of registers. Their location is simply an offset inside the I/O +/// region. For instance: +/// +/// ```ignore +/// register! { +/// pub FIXED_REG(u16) @ 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a 16-bit register named `FIXED_REG` located at offset `0x80` of an I/O region. +/// +/// These registers' location can be built simply by referencing their name: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// register! { +/// FIXED_REG(u32) @ 0x100 { +/// 15:8 high_byte; +/// 7:0 low_byte; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// let val = io.read(FIXED_REG); +/// +/// // Write from an already-existing value. +/// io.write(FIXED_REG, val.with_low_byte(0xff)); +/// +/// // Create a register value from scratch. +/// let val2 = FIXED_REG::zeroed().with_high_byte(0x80); +/// +/// // The location of fixed offset registers is already contained in their type. Thus, the +/// // `location` argument of `Io::write` is technically redundant and can be replaced by `()`. +/// io.write((), val2); +/// +/// // Or, the single-argument `Io::write_reg` can be used. +/// io.write_reg(val2); +/// # } +/// +/// ``` +/// +/// It is possible to create an alias of an existing register with new field definitions by using +/// the `=> ALIAS` syntax. This is useful for cases where a register's interpretation depends on +/// the context: +/// +/// ```no_run +/// use kernel::io::register; +/// +/// register! { +/// /// Scratch register. +/// pub SCRATCH(u32) @ 0x00000200 { +/// 31:0 value; +/// } +/// +/// /// Boot status of the firmware. +/// pub SCRATCH_BOOT_STATUS(u32) => SCRATCH { +/// 0:0 completed; +/// } +/// } +/// ``` +/// +/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing +/// its own `completed` field. +/// +/// ## Relative registers +/// +/// Relative registers can be instantiated several times at a relative offset of a group of bases. +/// For instance, imagine the following I/O space: +/// +/// ```text +/// +-----------------------------+ +/// | ... | +/// | | +/// 0x100--->+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ +/// ``` +/// +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done using the `Base + Offset` syntax when specifying the register's address: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REG(u32) @ Base + 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a register with an offset of `0x80` from a given base. +/// +/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the +/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for +/// this register needs to implement `RegisterBase`. +/// +/// The location of relative registers can be built using the [`WithBase::of`] method to specify +/// its base. All relative registers implement [`WithBase`]. +/// +/// Here is the above layout translated into code: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// // Type used to identify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. +/// register! { +/// /// CPU core control. +/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { +/// 0:0 start; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io.read(CPU_CTL::of::()); +/// +/// // Stop `Cpu0`. +/// io.write(WithBase::of::(), CPU_CTL::zeroed()); +/// # } +/// +/// // Aliases can also be defined for relative register. +/// register! { +/// /// Alias to CPU core control. +/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { +/// /// Start the aliased CPU core. +/// 1:1 alias_start; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { +/// // Start the aliased `CPU0`, leaving its other fields untouched. +/// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); +/// # } +/// ``` +/// +/// ## Arrays of registers +/// +/// Some I/O areas contain consecutive registers that share the same field layout. These areas can +/// be defined as an array of identical registers, allowing them to be accessed by index with +/// compile-time or runtime bound checking: +/// +/// ```ignore +/// register! { +/// pub REGISTER_ARRAY(u8)[10, stride = 4] @ 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// This defines `REGISTER_ARRAY`, an array of 10 byte registers starting at offset `0x100`. Each +/// register is separated from its neighbor by 4 bytes. +/// +/// The `stride` parameter is optional; if unspecified, the registers are placed consecutively from +/// each other. +/// +/// A location for a register in a register array is built using the [`Array::at`] trait method. +/// All arrays of registers implement [`Array`]. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::Array, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. +/// register! { +/// /// Scratch registers. +/// pub SCRATCH(u32)[64] @ 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) +/// # -> Result<(), Error>{ +/// // Read scratch register 0, i.e. I/O address `0x80`. +/// let scratch_0 = io.read(SCRATCH::at(0)).value(); +/// +/// // Write scratch register 15, i.e. I/O address `0x80 + (15 * 4)`. +/// io.write(Array::at(15), SCRATCH::from(0xffeeaabb)); +/// +/// // This is out of bounds and won't build. +/// // let scratch_128 = io.read(SCRATCH::at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let some_scratch = io.read(SCRATCH::try_at(idx).ok_or(EINVAL)?).value(); +/// +/// // Alias to a specific register in an array. +/// // Here `SCRATCH[8]` is used to convey the firmware exit code. +/// register! { +/// /// Firmware exit status code. +/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// let status = io.read(FIRMWARE_STATUS).status(); +/// +/// // Non-contiguous register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ 0x000000c4 { +/// 31:0 value; +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Relative arrays of registers +/// +/// Combining the two features described in the sections above, arrays of registers accessible from +/// a base can also be defined: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// Like relative registers, they implement the [`WithBase`] trait. However the return value of +/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the +/// [`at`](RelativeRegisterLoc::at) method. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Type used as parameter of `RegisterBase` to specify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // 64 per-cpu scratch registers, arranged as a contiguous array. +/// register! { +/// /// Per-CPU scratch registers. +/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// // Read scratch register 0 of CPU0. +/// let scratch = io.read(CPU_SCRATCH::of::().at(0)); +/// +/// // Write the retrieved value into scratch register 15 of CPU1. +/// io.write(WithBase::of::().at(15), scratch); +/// +/// // This won't build. +/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::().at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let scratch_idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let cpu0_scratch = io.read( +/// CPU_SCRATCH::of::().try_at(scratch_idx).ok_or(EINVAL)? +/// ).value(); +/// # Ok(()) +/// # } +/// +/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// register! { +/// /// Per-CPU firmware exit status code. +/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); +/// # Ok(()) +/// # } +/// ``` +#[macro_export] +macro_rules! register { + // Entry point for the macro, allowing multiple registers to be defined in one call. + // It matches all possible register declaration patterns to dispatch them to corresponding + // `@reg` rule that defines a single register. + // + // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O + // projections. + ( + $( + $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + $([ $size:expr $(, stride = $stride:expr)? ])? + $(@ $($base:ident +)? $offset:literal)? + $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? + { $($fields:tt)* } + )* + ) => { + $( + $crate::register!( + @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? + $(@ $($base +)? $offset)? + $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? + { $($fields)* } + ); + )* + }; + + // All the rules below are private helpers. + + // Creates a register at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an alias register of fixed offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates a register at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an alias register of relative offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an array of registers at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_array $name [ $size, stride = $stride ]); + }; + + // Shortcut for contiguous array of registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of array of registers `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] + { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an array of registers at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); + }; + + // Shortcut for contiguous array of relative registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $base + $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of relative array of registers `alias` with its own + // fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Generates the bitfield for the register. + // + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + ( + @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!( + #[allow(non_camel_case_types)] + $(#[$attr])* $vis struct $name($storage) { $($fields)* } + ); + }; + + // Implementations shared by all registers types. + (@io_base $name:ident($storage:ty) @ $offset:expr) => { + impl $crate::io::register::Register for $name { + type Storage = $storage; + + const OFFSET: usize = $offset; + } + }; + + // Implementations of fixed registers. + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { + impl $crate::io::register::FixedRegister for $name {} + + $(#[$attr])* + $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = + $crate::io::register::FixedRegisterLoc::<$name>::new(); + }; + + // Implementations of relative registers. + (@io_relative $name:ident @ $base:ident) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RelativeRegister for $name {} + }; + + // Implementations of register arrays. + (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { + impl $crate::io::register::Array for $name {} + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + }; + + // Implementations of relative array registers. + ( + @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident + ) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + + impl $crate::io::register::RelativeRegisterArray for $name {} + }; +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs new file mode 100644 index 000000000..17b0c174c --- /dev/null +++ b/rust/kernel/io/resource.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for [system +//! resources](https://docs.kernel.org/core-api/kernel-api.html#resources-management). +//! +//! C header: [`include/linux/ioport.h`](srctree/include/linux/ioport.h) + +use core::{ + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + str::CString, + types::Opaque, // +}; + +pub use super::{ + PhysAddr, + ResourceSize, // +}; + +/// A region allocated from a parent [`Resource`]. +/// +/// # Invariants +/// +/// - `self.0` points to a valid `bindings::resource` that was obtained through +/// `bindings::__request_region`. +pub struct Region { + /// The resource returned when the region was requested. + resource: NonNull, + /// The name that was passed in when the region was requested. We need to + /// store it for ownership reasons. + _name: CString, +} + +impl Deref for Region { + type Target = Resource; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as per the invariant of `Region`. + unsafe { Resource::from_raw(self.resource.as_ptr()) } + } +} + +impl Drop for Region { + fn drop(&mut self) { + let (flags, start, size) = { + let res = &**self; + (res.flags(), res.start(), res.size()) + }; + + let release_fn = if flags.contains(Flags::IORESOURCE_MEM) { + bindings::release_mem_region + } else { + bindings::release_region + }; + + // SAFETY: Safe as per the invariant of `Region`. + unsafe { release_fn(start, size) }; + } +} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, which is safe to be used from +// any thread. +unsafe impl Send for Region {} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, references to which are +// safe to be used from any thread. +unsafe impl Sync for Region {} + +/// A resource abstraction. +/// +/// # Invariants +/// +/// [`Resource`] is a transparent wrapper around a valid `bindings::resource`. +#[repr(transparent)] +pub struct Resource(Opaque); + +impl Resource { + /// Creates a reference to a [`Resource`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of 'a, the pointer will + /// point at a valid `bindings::resource`. + /// + /// The caller must also ensure that the [`Resource`] is only accessed via the + /// returned reference for the duration of 'a. + pub(crate) const unsafe fn from_raw<'a>(ptr: *mut bindings::resource) -> &'a Self { + // SAFETY: Self is a transparent wrapper around `Opaque`. + unsafe { &*ptr.cast() } + } + + /// Requests a resource region. + /// + /// Exclusive access will be given and the region will be marked as busy. + /// Further calls to [`Self::request_region`] will return [`None`] if + /// the region, or a part of it, is already in use. + pub fn request_region( + &self, + start: PhysAddr, + size: ResourceSize, + name: CString, + flags: Flags, + ) -> Option { + // SAFETY: + // - Safe as per the invariant of `Resource`. + // - `__request_region` will store a reference to the name, but that is + // safe as we own it and it will not be dropped until the `Region` is + // dropped. + let region = unsafe { + bindings::__request_region( + self.0.get(), + start, + size, + name.as_char_ptr(), + flags.0 as c_int, + ) + }; + + Some(Region { + resource: NonNull::new(region)?, + _name: name, + }) + } + + /// Returns the size of the resource. + pub fn size(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { bindings::resource_size(inner) } + } + + /// Returns the start address of the resource. + pub fn start(&self) -> PhysAddr { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { (*inner).start } + } + + /// Returns the name of the resource. + pub fn name(&self) -> Option<&CStr> { + let inner = self.0.get(); + + // SAFETY: Safe as per the invariants of `Resource`. + let name = unsafe { (*inner).name }; + + if name.is_null() { + return None; + } + + // SAFETY: In the C code, `resource::name` either contains a null + // pointer or points to a valid NUL-terminated C string, and at this + // point we know it is not null, so we can safely convert it to a + // `CStr`. + Some(unsafe { CStr::from_char_ptr(name) }) + } + + /// Returns the flags associated with the resource. + pub fn flags(&self) -> Flags { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + let flags = unsafe { (*inner).flags }; + + Flags(flags) + } +} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, which is +// safe to be used from any thread. +unsafe impl Send for Resource {} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, references +// to which are safe to be used from any thread. +unsafe impl Sync for Resource {} + +/// Resource flags as stored in the C `struct resource::flags` field. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the associated constants such as +/// [`Flags::IORESOURCE_IO`]. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +impl Flags { + /// PCI/ISA I/O ports. + pub const IORESOURCE_IO: Flags = Flags::new(bindings::IORESOURCE_IO); + + /// Resource is software muxed. + pub const IORESOURCE_MUXED: Flags = Flags::new(bindings::IORESOURCE_MUXED); + + /// Resource represents a memory region. + pub const IORESOURCE_MEM: Flags = Flags::new(bindings::IORESOURCE_MEM); + + /// Resource represents a memory region that must be ioremaped using `ioremap_np`. + pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn new(value: u32) -> Self { + build_assert!(value as u64 <= c_ulong::MAX as u64); + Flags(value as c_ulong) + } +} diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs new file mode 100644 index 000000000..5bb5b48cf --- /dev/null +++ b/rust/kernel/ioctl.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! `ioctl()` number definitions. +//! +//! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h) + +#![expect(non_snake_case)] + +use crate::build_assert::build_assert; + +/// Build an ioctl number, analogous to the C macro of the same name. +#[inline(always)] +const fn _IOC(dir: u32, ty: u32, nr: u32, size: usize) -> u32 { + build_assert!(dir <= uapi::_IOC_DIRMASK); + build_assert!(ty <= uapi::_IOC_TYPEMASK); + build_assert!(nr <= uapi::_IOC_NRMASK); + build_assert!(size <= (uapi::_IOC_SIZEMASK as usize)); + + (dir << uapi::_IOC_DIRSHIFT) + | (ty << uapi::_IOC_TYPESHIFT) + | (nr << uapi::_IOC_NRSHIFT) + | ((size as u32) << uapi::_IOC_SIZESHIFT) +} + +/// Build an ioctl number for an argumentless ioctl. +#[inline(always)] +pub const fn _IO(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_NONE, ty, nr, 0) +} + +/// Build an ioctl number for a read-only ioctl. +#[inline(always)] +pub const fn _IOR(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for a write-only ioctl. +#[inline(always)] +pub const fn _IOW(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for a read-write ioctl. +#[inline(always)] +pub const fn _IOWR(ty: u32, nr: u32) -> u32 { + _IOC( + uapi::_IOC_READ | uapi::_IOC_WRITE, + ty, + nr, + core::mem::size_of::(), + ) +} + +/// Get the ioctl direction from an ioctl number. +pub const fn _IOC_DIR(nr: u32) -> u32 { + (nr >> uapi::_IOC_DIRSHIFT) & uapi::_IOC_DIRMASK +} + +/// Get the ioctl type from an ioctl number. +pub const fn _IOC_TYPE(nr: u32) -> u32 { + (nr >> uapi::_IOC_TYPESHIFT) & uapi::_IOC_TYPEMASK +} + +/// Get the ioctl number from an ioctl number. +pub const fn _IOC_NR(nr: u32) -> u32 { + (nr >> uapi::_IOC_NRSHIFT) & uapi::_IOC_NRMASK +} + +/// Get the ioctl size from an ioctl number. +pub const fn _IOC_SIZE(nr: u32) -> usize { + ((nr >> uapi::_IOC_SIZESHIFT) & uapi::_IOC_SIZEMASK) as usize +} diff --git a/rust/kernel/iommu/mod.rs b/rust/kernel/iommu/mod.rs new file mode 100644 index 000000000..1423d7b19 --- /dev/null +++ b/rust/kernel/iommu/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust support related to IOMMU. + +pub mod pgtable; diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs new file mode 100644 index 000000000..5f9b42ca9 --- /dev/null +++ b/rust/kernel/iommu/pgtable.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IOMMU page table management. +//! +//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h) + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; + +use crate::{ + alloc, + bindings, + device::{ + Bound, + Device, // + }, + error::to_result, + io::PhysAddr, + prelude::*, // +}; + +use bindings::io_pgtable_fmt; + +/// Protection flags used with IOMMU mappings. +pub mod prot { + /// Read access. + pub const READ: u32 = bindings::IOMMU_READ; + /// Write access. + pub const WRITE: u32 = bindings::IOMMU_WRITE; + /// Request cache coherency. + pub const CACHE: u32 = bindings::IOMMU_CACHE; + /// Request no-execute permission. + pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; + /// MMIO peripheral mapping. + pub const MMIO: u32 = bindings::IOMMU_MMIO; + /// Privileged mapping. + pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV; +} + +/// Represents a requested `io_pgtable` configuration. +pub struct Config { + /// Quirk bitmask (type-specific). + pub quirks: usize, + /// Valid page sizes, as a bitmask of powers of two. + pub pgsize_bitmap: usize, + /// Input address space size in bits. + pub ias: u32, + /// Output address space size in bits. + pub oas: u32, + /// IOMMU uses coherent accesses for page table walks. + pub coherent_walk: bool, +} + +/// An io page table using a specific format. +/// +/// # Invariants +/// +/// The pointer references a valid io page table. +pub struct IoPageTable<'a, F: IoPageTableFmt> { + ptr: NonNull, + _dev: PhantomData<&'a Device>, + _marker: PhantomData, +} + +// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. +unsafe impl Send for IoPageTable<'_, F> {} +// SAFETY: `struct io_pgtable_ops` may be accessed concurrently. +unsafe impl Sync for IoPageTable<'_, F> {} + +/// The format used by this page table. +pub trait IoPageTableFmt: 'static { + /// The value representing this format. + const FORMAT: io_pgtable_fmt; +} + +impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { + /// Create a new `IoPageTable`. + #[inline] + pub fn new(dev: &'a Device, config: Config) -> Result> { + let mut raw_cfg = bindings::io_pgtable_cfg { + quirks: config.quirks, + pgsize_bitmap: config.pgsize_bitmap, + ias: config.ias, + oas: config.oas, + coherent_walk: config.coherent_walk, + tlb: &raw const NOOP_FLUSH_OPS, + iommu_dev: dev.as_raw(), + ..Zeroable::zeroed() + }; + + // SAFETY: + // * The raw_cfg pointer is valid for the duration of this call. + // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer + // as cookie. + // * The caller ensures that the io pgtable does not outlive the device. + let ops = unsafe { + bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut()) + }; + + // INVARIANT: We successfully created a valid page table. + Ok(IoPageTable { + ptr: NonNull::new(ops).ok_or(ENOMEM)?, + _dev: PhantomData, + _marker: PhantomData, + }) + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`. + #[inline] + pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops { + self.ptr.as_ptr() + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable`. + #[inline] + pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable { + // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable. + unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) } + } + + /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`. + #[inline] + pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg { + // SAFETY: The `raw_pgtable()` method returns a valid pointer. + unsafe { &raw mut (*self.raw_pgtable()).cfg } + } + + /// Map a physically contiguous range of pages of the same size. + /// + /// Even if successful, this operation may not map the entire range. In that case, only a + /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In + /// this case, the caller will usually call `map_pages` again for the remaining range. + /// + /// The returned [`Result`] indicates whether an error was encountered while mapping pages. + /// Note that this may return a non-zero length even if an error was encountered. The caller + /// will usually [unmap the relevant pages](Self::unmap_pages) on error. + /// + /// The caller must flush the TLB before using the pgtable to access the newly created mapping. + /// + /// # Safety + /// + /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while + /// this `map_pages` operation executes. + /// * This page table must not contain any mapping that overlaps with the mapping created by + /// this call. + /// * If this page table is live, then the caller must ensure that it's okay to access the + /// physical address being mapped for the duration in which it is mapped. + #[inline] + pub unsafe fn map_pages( + &self, + iova: usize, + paddr: PhysAddr, + pgsize: usize, + pgcount: usize, + prot: u32, + flags: alloc::Flags, + ) -> (usize, Result) { + let mut mapped: usize = 0; + + // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null. + let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() }; + + // SAFETY: The safety requirements of this method are sufficient to call `map_pages`. + let ret = to_result(unsafe { + (map_pages)( + self.raw_ops(), + iova, + paddr, + pgsize, + pgcount, + prot as i32, + flags.as_raw(), + &mut mapped, + ) + }); + + (mapped, ret) + } + + /// Unmap a range of virtually contiguous pages of the same size. + /// + /// This may not unmap the entire range, and returns the length of the unmapped prefix in + /// bytes. + /// + /// # Safety + /// + /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while + /// this `unmap_pages` operation executes. + /// * This page table must contain one or more consecutive mappings starting at `iova` whose + /// total size is `pgcount * pgsize`. + #[inline] + #[must_use] + pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize { + // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null. + let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() }; + + // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`. + unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) } + } +} + +// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all +// required invalidations using a range. There is no need for it get ARM style invalidation +// instructions from the page table code. +// +// Support for flushing the TLB with ARM style invalidation instructions may be added in the +// future. +static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { + tlb_flush_all: Some(rust_tlb_flush_all_noop), + tlb_flush_walk: Some(rust_tlb_flush_walk_noop), + tlb_add_page: None, +}; + +#[no_mangle] +extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {} + +#[no_mangle] +extern "C" fn rust_tlb_flush_walk_noop( + _iova: usize, + _size: usize, + _granule: usize, + _cookie: *mut core::ffi::c_void, +) { +} + +impl Drop for IoPageTable<'_, F> { + fn drop(&mut self) { + // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this + // destructor runs. + unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) }; + } +} + +/// The `ARM_64_LPAE_S1` page table format. +pub enum ARM64LPAES1 {} + +impl IoPageTableFmt for ARM64LPAES1 { + const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; +} + +impl IoPageTable<'_, ARM64LPAES1> { + /// Access the `ttbr` field of the configuration. + /// + /// This is the physical address of the page table, which may be passed to the device that + /// needs to use it. + /// + /// # Safety + /// + /// The caller must ensure that the device stops using the page table before dropping it. + #[inline] + pub unsafe fn ttbr(&self) -> u64 { + // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. + unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr } + } + + /// Access the `mair` field of the configuration. + #[inline] + pub fn mair(&self) -> u64 { + // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. + unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair } + } +} diff --git a/rust/kernel/iov.rs b/rust/kernel/iov.rs new file mode 100644 index 000000000..43bae8923 --- /dev/null +++ b/rust/kernel/iov.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! IO vectors. +//! +//! C headers: [`include/linux/iov_iter.h`](srctree/include/linux/iov_iter.h), +//! [`include/linux/uio.h`](srctree/include/linux/uio.h) + +use crate::{ + alloc::{Allocator, Flags}, + bindings, + prelude::*, + types::Opaque, +}; +use core::{marker::PhantomData, mem::MaybeUninit, ptr, slice}; + +const ITER_SOURCE: bool = bindings::ITER_SOURCE != 0; +const ITER_DEST: bool = bindings::ITER_DEST != 0; + +// Compile-time assertion for the above constants. +const _: () = { + build_assert!( + ITER_SOURCE != ITER_DEST, + "ITER_DEST and ITER_SOURCE should be different." + ); +}; + +/// An IO vector that acts as a source of data. +/// +/// The data may come from many different sources. This includes both things in kernel-space and +/// reading from userspace. It's not necessarily the case that the data source is immutable, so +/// rewinding the IO vector to read the same data twice is not guaranteed to result in the same +/// bytes. It's also possible that the data source is mapped in a thread-local manner using e.g. +/// `kmap_local_page()`, so this type is not `Send` to ensure that the mapping is read from the +/// right context in that scenario. +/// +/// # Invariants +/// +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_SOURCE`. For the duration +/// of `'data`, it must be safe to read from this IO vector using the standard C methods for this +/// purpose. +#[repr(transparent)] +pub struct IovIterSource<'data> { + iov: Opaque, + /// Represent to the type system that this value contains a pointer to readable data it does + /// not own. + _source: PhantomData<&'data [u8]>, +} + +impl<'data> IovIterSource<'data> { + /// Obtain an `IovIterSource` from a raw pointer. + /// + /// # Safety + /// + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the + /// returned reference for the duration of `'iov`. + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_SOURCE`. + /// * For the duration of `'data`, it must be safe to read from this IO vector using the + /// standard C methods for this purpose. + #[track_caller] + #[inline] + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterSource<'data> { + // SAFETY: The caller ensures that `ptr` is valid. + let data_source = unsafe { (*ptr).data_source }; + assert_eq!(data_source, ITER_SOURCE); + + // SAFETY: The caller ensures the type invariants for the right durations, and + // `IovIterSource` is layout compatible with `struct iov_iter`. + unsafe { &mut *ptr.cast::>() } + } + + /// Access this as a raw `struct iov_iter`. + #[inline] + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { + self.iov.get() + } + + /// Returns the number of bytes available in this IO vector. + /// + /// Note that this may overestimate the number of bytes. For example, reading from userspace + /// memory could fail with `EFAULT`, which will be treated as the end of the IO vector. + #[inline] + pub fn len(&self) -> usize { + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. + unsafe { + (*self.iov.get()) + .__bindgen_anon_1 + .__bindgen_anon_1 + .as_ref() + .count + } + } + + /// Returns whether there are any bytes left in this IO vector. + /// + /// This may return `true` even if there are no more bytes available. For example, reading from + /// userspace memory could fail with `EFAULT`, which will be treated as the end of the IO vector. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Advance this IO vector by `bytes` bytes. + /// + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. + #[inline] + pub fn advance(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; + } + + /// Advance this IO vector backwards by `bytes` bytes. + /// + /// # Safety + /// + /// The IO vector must not be reverted to before its beginning. + #[inline] + pub unsafe fn revert(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller + // ensures that `bytes` is in bounds. + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; + } + + /// Read data from this IO vector. + /// + /// Returns the number of bytes that have been copied. + #[inline] + pub fn copy_from_iter(&mut self, out: &mut [u8]) -> usize { + // SAFETY: `Self::copy_from_iter_raw` guarantees that it will not write any uninitialized + // bytes in the provided buffer, so `out` is still a valid `u8` slice after this call. + let out = unsafe { &mut *(ptr::from_mut(out) as *mut [MaybeUninit]) }; + + self.copy_from_iter_raw(out).len() + } + + /// Read data from this IO vector and append it to a vector. + /// + /// Returns the number of bytes that have been copied. + #[inline] + pub fn copy_from_iter_vec( + &mut self, + out: &mut Vec, + flags: Flags, + ) -> Result { + out.reserve(self.len(), flags)?; + let len = self.copy_from_iter_raw(out.spare_capacity_mut()).len(); + // SAFETY: + // - `len` is the length of a subslice of the spare capacity, so `len` is at most the + // length of the spare capacity. + // - `Self::copy_from_iter_raw` guarantees that the first `len` bytes of the spare capacity + // have been initialized. + unsafe { out.inc_len(len) }; + Ok(len) + } + + /// Read data from this IO vector into potentially uninitialized memory. + /// + /// Returns the sub-slice of the output that has been initialized. If the returned slice is + /// shorter than the input buffer, then the entire IO vector has been read. + /// + /// This will never write uninitialized bytes to the provided buffer. + #[inline] + pub fn copy_from_iter_raw(&mut self, out: &mut [MaybeUninit]) -> &mut [u8] { + let capacity = out.len(); + let out = out.as_mut_ptr().cast::(); + + // GUARANTEES: The C API guarantees that it does not write uninitialized bytes to the + // provided buffer. + // SAFETY: + // * By the type invariants, it is still valid to read from this IO vector. + // * `out` is valid for writing for `capacity` bytes because it comes from a slice of + // that length. + let len = unsafe { bindings::_copy_from_iter(out.cast(), capacity, self.as_raw()) }; + + // SAFETY: The underlying C api guarantees that initialized bytes have been written to the + // first `len` bytes of the spare capacity. + unsafe { slice::from_raw_parts_mut(out, len) } + } +} + +/// An IO vector that acts as a destination for data. +/// +/// IO vectors support many different types of destinations. This includes both buffers in +/// kernel-space and writing to userspace. It's possible that the destination buffer is mapped in a +/// thread-local manner using e.g. `kmap_local_page()`, so this type is not `Send` to ensure that +/// the mapping is written to the right context in that scenario. +/// +/// # Invariants +/// +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_DEST`. For the duration of +/// `'data`, it must be safe to write to this IO vector using the standard C methods for this +/// purpose. +#[repr(transparent)] +pub struct IovIterDest<'data> { + iov: Opaque, + /// Represent to the type system that this value contains a pointer to writable data it does + /// not own. + _source: PhantomData<&'data mut [u8]>, +} + +impl<'data> IovIterDest<'data> { + /// Obtain an `IovIterDest` from a raw pointer. + /// + /// # Safety + /// + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the + /// returned reference for the duration of `'iov`. + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_DEST`. + /// * For the duration of `'data`, it must be safe to write to this IO vector using the + /// standard C methods for this purpose. + #[track_caller] + #[inline] + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterDest<'data> { + // SAFETY: The caller ensures that `ptr` is valid. + let data_source = unsafe { (*ptr).data_source }; + assert_eq!(data_source, ITER_DEST); + + // SAFETY: The caller ensures the type invariants for the right durations, and + // `IovIterSource` is layout compatible with `struct iov_iter`. + unsafe { &mut *ptr.cast::>() } + } + + /// Access this as a raw `struct iov_iter`. + #[inline] + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { + self.iov.get() + } + + /// Returns the number of bytes available in this IO vector. + /// + /// Note that this may overestimate the number of bytes. For example, reading from userspace + /// memory could fail with EFAULT, which will be treated as the end of the IO vector. + #[inline] + pub fn len(&self) -> usize { + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. + unsafe { + (*self.iov.get()) + .__bindgen_anon_1 + .__bindgen_anon_1 + .as_ref() + .count + } + } + + /// Returns whether there are any bytes left in this IO vector. + /// + /// This may return `true` even if there are no more bytes available. For example, reading from + /// userspace memory could fail with EFAULT, which will be treated as the end of the IO vector. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Advance this IO vector by `bytes` bytes. + /// + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. + #[inline] + pub fn advance(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; + } + + /// Advance this IO vector backwards by `bytes` bytes. + /// + /// # Safety + /// + /// The IO vector must not be reverted to before its beginning. + #[inline] + pub unsafe fn revert(&mut self, bytes: usize) { + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller + // ensures that `bytes` is in bounds. + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; + } + + /// Write data to this IO vector. + /// + /// Returns the number of bytes that were written. If this is shorter than the provided slice, + /// then no more bytes can be written. + #[inline] + pub fn copy_to_iter(&mut self, input: &[u8]) -> usize { + // SAFETY: + // * By the type invariants, it is still valid to write to this IO vector. + // * `input` is valid for `input.len()` bytes. + unsafe { bindings::_copy_to_iter(input.as_ptr().cast(), input.len(), self.as_raw()) } + } + + /// Utility for implementing `read_iter` given the full contents of the file. + /// + /// The full contents of the file being read from is represented by `contents`. This call will + /// write the appropriate sub-slice of `contents` and update the file position in `ppos` so + /// that the file will appear to contain `contents` even if takes multiple reads to read the + /// entire file. + #[inline] + pub fn simple_read_from_buffer(&mut self, ppos: &mut i64, contents: &[u8]) -> Result { + if *ppos < 0 { + return Err(EINVAL); + } + let Ok(pos) = usize::try_from(*ppos) else { + return Ok(0); + }; + if pos >= contents.len() { + return Ok(0); + } + + // BOUNDS: We just checked that `pos < contents.len()` above. + let num_written = self.copy_to_iter(&contents[pos..]); + + // OVERFLOW: `pos+num_written <= contents.len() <= isize::MAX <= i64::MAX`. + *ppos = (pos + num_written) as i64; + + Ok(num_written) + } +} diff --git a/rust/kernel/irq.rs b/rust/kernel/irq.rs new file mode 100644 index 000000000..09ef1e7f8 --- /dev/null +++ b/rust/kernel/irq.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IRQ abstractions. +//! +//! An IRQ is an interrupt request from a device. It is used to get the CPU's +//! attention so it can service a hardware event in a timely manner. +//! +//! The current abstractions handle IRQ requests and handlers, i.e.: it allows +//! drivers to register a handler for a given IRQ line. +//! +//! C header: [`include/linux/interrupt.h`](srctree/include/linux/interrupt.h) + +/// Flags to be used when registering IRQ handlers. +mod flags; + +/// IRQ allocation and handling. +mod request; + +pub use flags::Flags; + +pub use request::{ + Handler, IrqRequest, IrqReturn, Registration, ThreadedHandler, ThreadedIrqReturn, + ThreadedRegistration, +}; diff --git a/rust/kernel/irq/flags.rs b/rust/kernel/irq/flags.rs new file mode 100644 index 000000000..d26e25af0 --- /dev/null +++ b/rust/kernel/irq/flags.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright 2025 Collabora ltd. + +use crate::bindings; +use crate::prelude::*; + +/// Flags to be used when registering IRQ handlers. +/// +/// Flags can be used to request specific behaviors when registering an IRQ +/// handler, and can be combined using the `|`, `&`, and `!` operators to +/// further control the system's behavior. +/// +/// A common use case is to register a shared interrupt, as sharing the line +/// between devices is increasingly common in modern systems and is even +/// required for some buses. This requires setting [`Flags::SHARED`] when +/// requesting the interrupt. Other use cases include setting the trigger type +/// through `Flags::TRIGGER_*`, which determines when the interrupt fires, or +/// controlling whether the interrupt is masked after the handler runs by using +/// [`Flags::ONESHOT`]. +/// +/// If an invalid combination of flags is provided, the system will refuse to +/// register the handler, and lower layers will enforce certain flags when +/// necessary. This means, for example, that all the +/// [`crate::irq::Registration`] for a shared interrupt have to agree on +/// [`Flags::SHARED`] and on the same trigger type, if set. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Use the interrupt line as already configured. + pub const TRIGGER_NONE: Flags = Flags::new(bindings::IRQF_TRIGGER_NONE); + + /// The interrupt is triggered when the signal goes from low to high. + pub const TRIGGER_RISING: Flags = Flags::new(bindings::IRQF_TRIGGER_RISING); + + /// The interrupt is triggered when the signal goes from high to low. + pub const TRIGGER_FALLING: Flags = Flags::new(bindings::IRQF_TRIGGER_FALLING); + + /// The interrupt is triggered while the signal is held high. + pub const TRIGGER_HIGH: Flags = Flags::new(bindings::IRQF_TRIGGER_HIGH); + + /// The interrupt is triggered while the signal is held low. + pub const TRIGGER_LOW: Flags = Flags::new(bindings::IRQF_TRIGGER_LOW); + + /// Allow sharing the IRQ among several devices. + pub const SHARED: Flags = Flags::new(bindings::IRQF_SHARED); + + /// Set by callers when they expect sharing mismatches to occur. + pub const PROBE_SHARED: Flags = Flags::new(bindings::IRQF_PROBE_SHARED); + + /// Flag to mark this interrupt as timer interrupt. + pub const TIMER: Flags = Flags::new(bindings::IRQF_TIMER); + + /// Interrupt is per CPU. + pub const PERCPU: Flags = Flags::new(bindings::IRQF_PERCPU); + + /// Flag to exclude this interrupt from irq balancing. + pub const NOBALANCING: Flags = Flags::new(bindings::IRQF_NOBALANCING); + + /// Interrupt is used for polling (only the interrupt that is registered + /// first in a shared interrupt is considered for performance reasons). + pub const IRQPOLL: Flags = Flags::new(bindings::IRQF_IRQPOLL); + + /// Interrupt is not re-enabled after the hardirq handler finished. Used by + /// threaded interrupts which need to keep the irq line disabled until the + /// threaded handler has been run. + pub const ONESHOT: Flags = Flags::new(bindings::IRQF_ONESHOT); + + /// Do not disable this IRQ during suspend. Does not guarantee that this + /// interrupt will wake the system from a suspended state. + pub const NO_SUSPEND: Flags = Flags::new(bindings::IRQF_NO_SUSPEND); + + /// Force enable it on resume even if [`Flags::NO_SUSPEND`] is set. + pub const FORCE_RESUME: Flags = Flags::new(bindings::IRQF_FORCE_RESUME); + + /// Interrupt cannot be threaded. + pub const NO_THREAD: Flags = Flags::new(bindings::IRQF_NO_THREAD); + + /// Resume IRQ early during syscore instead of at device resume time. + pub const EARLY_RESUME: Flags = Flags::new(bindings::IRQF_EARLY_RESUME); + + /// If the IRQ is shared with a [`Flags::NO_SUSPEND`] user, execute this + /// interrupt handler after suspending interrupts. For system wakeup devices + /// users need to implement wakeup detection in their interrupt handlers. + pub const COND_SUSPEND: Flags = Flags::new(bindings::IRQF_COND_SUSPEND); + + /// Don't enable IRQ or NMI automatically when users request it. Users will + /// enable it explicitly by `enable_irq` or `enable_nmi` later. + pub const NO_AUTOEN: Flags = Flags::new(bindings::IRQF_NO_AUTOEN); + + /// Exclude from runnaway detection for IPI and similar handlers, depends on + /// `PERCPU`. + pub const NO_DEBUG: Flags = Flags::new(bindings::IRQF_NO_DEBUG); + + pub(crate) fn into_inner(self) -> c_ulong { + self.0 + } + + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + const fn new(value: u32) -> Self { + build_assert!(value as u64 <= c_ulong::MAX as u64); + Self(value as c_ulong) + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs new file mode 100644 index 000000000..c1c6525a6 --- /dev/null +++ b/rust/kernel/irq/request.rs @@ -0,0 +1,483 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright 2025 Collabora ltd. + +//! This module provides types like [`Registration`] and +//! [`ThreadedRegistration`], which allow users to register handlers for a given +//! IRQ line. + +use core::marker::{ + PhantomData, + PhantomPinned, // +}; + +use crate::{ + device::{ + Bound, + Device, // + }, + error::to_result, + irq::flags::Flags, + prelude::*, + str::CStr, +}; + +/// The value that can be returned from a [`Handler`] or a [`ThreadedHandler`]. +#[repr(u32)] +pub enum IrqReturn { + /// The interrupt was not from this device or was not handled. + None = bindings::irqreturn_IRQ_NONE, + + /// The interrupt was handled by this device. + Handled = bindings::irqreturn_IRQ_HANDLED, +} + +/// Callbacks for an IRQ handler. +pub trait Handler: Sync { + /// The hard IRQ handler. + /// + /// This is executed in interrupt context, hence all corresponding + /// limitations do apply. + /// + /// All work that does not necessarily need to be executed from + /// interrupt context, should be deferred to a threaded handler. + /// See also [`ThreadedRegistration`]. + fn handle(&self) -> IrqReturn; +} + +/// A request for an IRQ line for a given device. +/// +/// # Invariants +/// +/// - `ìrq` is the number of an interrupt source of `dev`. +/// - `irq` has not been registered yet; this is consumed by [`Registration::new()`]. +pub struct IrqRequest<'a> { + irq: u32, + /// Proves the device is bound at registration time and ties `'a` to the device's bound + /// lifetime, ensuring the [`Registration`] cannot outlive it. + _dev: PhantomData<&'a Device>, +} + +impl<'a> IrqRequest<'a> { + /// Creates a new IRQ request for the given device and IRQ number. + /// + /// # Safety + /// + /// - `irq` should be a valid IRQ number for `dev`. + pub(crate) unsafe fn new(_dev: &'a Device, irq: u32) -> Self { + // INVARIANT: `irq` is a valid IRQ number for `dev`. + IrqRequest { + irq, + _dev: PhantomData, + } + } + + /// Returns the IRQ number of an [`IrqRequest`]. + #[inline] + pub fn irq(&self) -> u32 { + self.irq + } +} + +/// A registration of an IRQ handler for a given IRQ line. +/// +/// # Examples +/// +/// The following is an example of using `Registration`. It uses a +/// [`Completion`] to coordinate between the IRQ +/// handler and process context. [`Completion`] uses interior mutability, so the +/// handler can signal with [`Completion::complete_all()`] and the process +/// context can wait with [`Completion::wait_for_completion()`] even though +/// there is no way to get a mutable reference to the any of the fields in +/// `Data`. +/// +/// [`Completion`]: kernel::sync::Completion +/// [`Completion::complete_all()`]: kernel::sync::Completion::complete_all +/// [`Completion::wait_for_completion()`]: kernel::sync::Completion::wait_for_completion +/// +/// ``` +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// Registration, +/// }, +/// prelude::*, +/// sync::Completion, +/// }; +/// +/// // Data shared between process and IRQ context. +/// #[pin_data] +/// struct Data { +/// #[pin] +/// completion: Completion, +/// } +/// +/// impl irq::Handler for Data { +/// // Executed in IRQ context. +/// fn handle(&self) -> IrqReturn { +/// self.completion.complete_all(); +/// IrqReturn::Handled +/// } +/// } +/// +/// // Registers an IRQ handler for the given IrqRequest. +/// // +/// // This runs in process context and assumes `request` was previously acquired from a device. +/// fn register_irq( +/// request: IrqRequest<'_>, +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// Registration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// completion <- Completion::new(), +/// }? Error), +/// ) +/// }; +/// +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; +/// +/// registration.handler().completion.wait_for_completion(); +/// +/// Ok(registration) +/// } +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * We own an irq handler registered via `request_irq` whose cookie is a pointer to `Self`. +#[pin_data(PinnedDrop)] +pub struct Registration<'a, T: Handler> { + request: IrqRequest<'a>, + + #[pin] + handler: T, + + /// Pinned because we need address stability so that we can pass a pointer + /// to the callback. + #[pin] + _pin: PhantomPinned, +} + +impl<'a, T: Handler> Registration<'a, T> { + /// Registers the IRQ handler with the system for the given IRQ number. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn new( + request: IrqRequest<'a>, + flags: Flags, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_irq` whose cookie is a pointer to `Self`. + try_pin_init!(&this in Self { + handler <- handler, + request, + _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callback to be + // called immediately. + to_result(unsafe { + bindings::request_irq( + request.irq, + Some(handle_irq_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, + }) + } + + /// Returns a reference to the handler that was registered with the system. + pub fn handler(&self) -> &T { + &self.handler + } + + /// Wait for pending IRQ handlers on other CPUs. + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `Registration::new()`. This blocks + // until all in-flight handlers complete, so no references to `self` remain after this + // returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; + } +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_irq`. +unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. + let registration = unsafe { &*ptr }; + + T::handle(®istration.handler) as c_uint +} + +/// The value that can be returned from [`ThreadedHandler::handle`]. +#[repr(u32)] +pub enum ThreadedIrqReturn { + /// The interrupt was not from this device or was not handled. + None = bindings::irqreturn_IRQ_NONE, + + /// The interrupt was handled by this device. + Handled = bindings::irqreturn_IRQ_HANDLED, + + /// The handler wants the handler thread to wake up. + WakeThread = bindings::irqreturn_IRQ_WAKE_THREAD, +} + +/// Callbacks for a threaded IRQ handler. +pub trait ThreadedHandler: Sync { + /// The hard IRQ handler. + /// + /// This is executed in interrupt context, hence all corresponding + /// limitations do apply. All work that does not necessarily need to be + /// executed from interrupt context, should be deferred to the threaded + /// handler, i.e. [`ThreadedHandler::handle_threaded`]. + /// + /// The default implementation returns [`ThreadedIrqReturn::WakeThread`]. + fn handle(&self) -> ThreadedIrqReturn { + ThreadedIrqReturn::WakeThread + } + + /// The threaded IRQ handler. + /// + /// This is executed in process context. The kernel creates a dedicated + /// `kthread` for this purpose. + fn handle_threaded(&self) -> IrqReturn; +} + +/// A registration of a threaded IRQ handler for a given IRQ line. +/// +/// Two callbacks are required: one to handle the IRQ, and one to handle any +/// other work in a separate thread. +/// +/// The thread handler is only called if the IRQ handler returns +/// [`ThreadedIrqReturn::WakeThread`]. +/// +/// # Examples +/// +/// The following is an example of using [`ThreadedRegistration`]. It uses a +/// [`Mutex`](kernel::sync::Mutex) to provide interior mutability. +/// +/// ``` +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// ThreadedHandler, +/// ThreadedIrqReturn, +/// ThreadedRegistration, +/// }, +/// prelude::*, +/// sync::Mutex, +/// }; +/// +/// // Declare a struct that will be passed in when the interrupt fires. The u32 +/// // merely serves as an example of some internal data. +/// // +/// // [`irq::ThreadedHandler::handle`] takes `&self`. This example +/// // illustrates how interior mutability can be used when sharing the data +/// // between process context and IRQ context. +/// #[pin_data] +/// struct Data { +/// #[pin] +/// value: Mutex, +/// } +/// +/// impl ThreadedHandler for Data { +/// // This will run (in a separate kthread) if and only if +/// // [`ThreadedHandler::handle`] returns [`WakeThread`], which it does by +/// // default. +/// fn handle_threaded(&self) -> IrqReturn { +/// let mut data = self.value.lock(); +/// *data += 1; +/// IrqReturn::Handled +/// } +/// } +/// +/// // Registers a threaded IRQ handler for the given [`IrqRequest`]. +/// // +/// // This is executing in process context and assumes that `request` was +/// // previously acquired from a device. +/// fn register_threaded_irq( +/// request: IrqRequest<'_>, +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// ThreadedRegistration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// value <- kernel::new_mutex!(0), +/// }? Error), +/// ) +/// }; +/// +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; +/// +/// { +/// // The data can be accessed from process context too. +/// let mut data = registration.handler().value.lock(); +/// *data += 1; +/// } +/// +/// Ok(registration) +/// } +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// * We own an irq handler registered via `request_threaded_irq` whose cookie is a pointer to +/// `Self`. +#[pin_data(PinnedDrop)] +pub struct ThreadedRegistration<'a, T: ThreadedHandler> { + request: IrqRequest<'a>, + + #[pin] + handler: T, + + /// Pinned because we need address stability so that we can pass a pointer + /// to the callback. + #[pin] + _pin: PhantomPinned, +} + +impl<'a, T: ThreadedHandler> ThreadedRegistration<'a, T> { + /// Registers the IRQ handler with the system for the given IRQ number. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`ThreadedRegistration`] or otherwise prevent + /// its [`Drop`] implementation from running. + pub unsafe fn new( + request: IrqRequest<'a>, + flags: Flags, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_threaded_irq` whose cookie is a pointer to `Self`. + try_pin_init!(&this in Self { + handler <- handler, + request, + _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_threaded_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callbacks to be + // called immediately. + to_result(unsafe { + bindings::request_threaded_irq( + request.irq, + Some(handle_threaded_irq_callback::), + Some(thread_fn_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, + }) + } + + /// Returns a reference to the handler that was registered with the system. + pub fn handler(&self) -> &T { + &self.handler + } + + /// Wait for pending IRQ handlers on other CPUs. + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; + } +} + +#[pinned_drop] +impl PinnedDrop for ThreadedRegistration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `ThreadedRegistration::new()`. This + // blocks until all in-flight handlers complete, so no references to `self` remain after + // this returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; + } +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_threaded_irq`. +unsafe extern "C" fn handle_threaded_irq_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; + + T::handle(®istration.handler) as c_uint +} + +/// # Safety +/// +/// This function should be only used as the callback in `request_threaded_irq`. +unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; + + T::handle_threaded(®istration.handler) as c_uint +} diff --git a/rust/kernel/jump_label.rs b/rust/kernel/jump_label.rs new file mode 100644 index 000000000..f54cedcb6 --- /dev/null +++ b/rust/kernel/jump_label.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for static keys. +//! +//! C header: [`include/linux/jump_label.h`](srctree/include/linux/jump_label.h). + +/// Branch based on a static key. +/// +/// Takes three arguments: +/// +/// * `key` - the path to the static variable containing the `static_key`. +/// * `keytyp` - the type of `key`. +/// * `field` - the name of the field of `key` that contains the `static_key`. +/// +/// # Safety +/// +/// The macro must be used with a real static key defined by C. +#[macro_export] +macro_rules! static_branch_unlikely { + ($key:path, $keytyp:ty, $field:ident) => {{ + let _key: *const $keytyp = ::core::ptr::addr_of!($key); + let _key: *const $crate::bindings::static_key_false = ::core::ptr::addr_of!((*_key).$field); + let _key: *const $crate::bindings::static_key = _key.cast(); + + #[cfg(not(CONFIG_JUMP_LABEL))] + { + $crate::bindings::static_key_count(_key.cast_mut()) > 0 + } + + #[cfg(CONFIG_JUMP_LABEL)] + $crate::jump_label::arch_static_branch! { $key, $keytyp, $field, false } + }}; +} +pub use static_branch_unlikely; + +/// Assert that the assembly block evaluates to a string literal. +#[cfg(CONFIG_JUMP_LABEL)] +const _: &str = include!(concat!( + env!("OBJTREE"), + "/rust/kernel/generated_arch_static_branch_asm.rs" +)); + +#[macro_export] +#[doc(hidden)] +#[cfg(not(testlib))] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: { + $crate::asm!( + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_static_branch_asm.rs")); + l_yes = label { + break 'my_label true; + }, + symb = sym $key, + off = const ::core::mem::offset_of!($keytyp, $field), + branch = const $crate::jump_label::bool_to_int($branch), + ); + + break 'my_label false; + }}; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(testlib)] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => { + // The asm falls through until patched, which never happens on the host. + false + }; +} + +#[cfg(CONFIG_JUMP_LABEL)] +pub use arch_static_branch; + +/// A helper used by inline assembly to pass a boolean to as a `const` parameter. +/// +/// Using this function instead of a cast lets you assert that the input is a boolean, and not some +/// other type that can also be cast to an integer. +#[doc(hidden)] +pub const fn bool_to_int(b: bool) -> i32 { + b as i32 +} diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs new file mode 100644 index 000000000..91eaff8c1 --- /dev/null +++ b/rust/kernel/kunit.rs @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! KUnit-based macros for Rust unit tests. +//! +//! C header: [`include/kunit/test.h`](srctree/include/kunit/test.h) +//! +//! Reference: + +use crate::fmt; +use crate::prelude::*; + +/// Prints a KUnit error-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn err(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + c"\x013%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Prints a KUnit info-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn info(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + c"\x016%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Asserts that a boolean expression is `true` at runtime. +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert { + ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + 'out: { + // Do nothing if the condition is `true`. + if $condition { + break 'out; + } + + static FILE: &'static $crate::str::CStr = $file; + static LINE: i32 = ::core::line!() as i32 - $diff; + static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); + + // SAFETY: FFI call without safety requirements. + let kunit_test = unsafe { $crate::bindings::kunit_get_current_test() }; + if kunit_test.is_null() { + // The assertion failed but this task is not running a KUnit test, so we cannot call + // KUnit, but at least print an error to the kernel log. This may happen if this + // macro is called from an spawned thread in a test (see + // `scripts/rustdoc_test_gen.rs`) or if some non-test code calls this macro by + // mistake (it is hidden to prevent that). + // + // This mimics KUnit's failed assertion format. + $crate::kunit::err($crate::prelude::fmt!( + " # {}: ASSERTION FAILED at {FILE}:{LINE}\n", + $name + )); + $crate::kunit::err($crate::prelude::fmt!( + " Expected {CONDITION} to be true, but is false\n" + )); + $crate::kunit::err($crate::prelude::fmt!( + " Failure not reported to KUnit since this is a non-KUnit task\n" + )); + break 'out; + } + + #[repr(transparent)] + struct Location($crate::bindings::kunit_loc); + + #[repr(transparent)] + struct UnaryAssert($crate::bindings::kunit_unary_assert); + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for Location {} + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for UnaryAssert {} + + static LOCATION: Location = Location($crate::bindings::kunit_loc { + file: $crate::str::as_char_ptr_in_const_context(FILE), + line: LINE, + }); + static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert { + assert: $crate::bindings::kunit_assert {}, + condition: $crate::str::as_char_ptr_in_const_context(CONDITION), + expected_true: true, + }); + + // SAFETY: + // - FFI call. + // - The `kunit_test` pointer is valid because we got it from + // `kunit_get_current_test()` and it was not null. This means we are in a KUnit + // test, and that the pointer can be passed to KUnit functions and assertions. + // - The string pointers (`file` and `condition` above) point to null-terminated + // strings since they are `CStr`s. + // - The function pointer (`format`) points to the proper function. + // - The pointers passed will remain valid since they point to `static`s. + // - The format string is allowed to be null. + // - There are, however, problems with this: first of all, this will end up stopping + // the thread, without running destructors. While that is problematic in itself, + // it is considered UB to have what is effectively a forced foreign unwind + // with `extern "C"` ABI. One could observe the stack that is now gone from + // another thread. We should avoid pinning stack variables to prevent library UB, + // too. For the moment, given that test failures are reported immediately before the + // next test runs, that test failures should be fixed and that KUnit is explicitly + // documented as not suitable for production environments, we feel it is reasonable. + unsafe { + $crate::bindings::__kunit_do_failed_assertion( + kunit_test, + ::core::ptr::addr_of!(LOCATION.0), + $crate::bindings::kunit_assert_type_KUNIT_ASSERTION, + ::core::ptr::addr_of!(ASSERTION.0.assert), + Some($crate::bindings::kunit_unary_assert_format), + ::core::ptr::null(), + ); + } + + // SAFETY: FFI call; the `test` pointer is valid because this hidden macro should only + // be called by the generated documentation tests which forward the test pointer given + // by KUnit. + unsafe { + $crate::bindings::__kunit_abort(kunit_test); + } + } + }; +} + +/// Asserts that two expressions are equal to each other (using [`PartialEq`]). +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert_eq { + ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + // For the moment, we just forward to the expression assert because, for binary asserts, + // KUnit supports only a few types (e.g. integers). + $crate::kunit_assert!($name, $file, $diff, $left == $right); + }}; +} + +trait TestResult { + fn is_test_result_ok(&self) -> bool; +} + +impl TestResult for () { + fn is_test_result_ok(&self) -> bool { + true + } +} + +impl TestResult for Result { + fn is_test_result_ok(&self) -> bool { + self.is_ok() + } +} + +/// Returns whether a test result is to be considered OK. +/// +/// This will be `assert!`ed from the generated tests. +#[doc(hidden)] +#[expect(private_bounds)] +pub fn is_test_result_ok(t: impl TestResult) -> bool { + t.is_test_result_ok() +} + +/// Represents an individual test case. +#[doc(hidden)] +pub const fn kunit_case( + name: &'static kernel::str::CStr, + run_case: unsafe extern "C" fn(*mut kernel::bindings::kunit), +) -> kernel::bindings::kunit_case { + kernel::bindings::kunit_case { + run_case: Some(run_case), + name: kernel::str::as_char_ptr_in_const_context(name), + attr: kernel::bindings::kunit_attributes { + speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + generate_params: None, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + module_name: core::ptr::null_mut(), + log: core::ptr::null_mut(), + param_init: None, + param_exit: None, + } +} + +/// Registers a KUnit test suite. +/// +/// # Safety +/// +/// `test_cases` must be a `NULL` terminated array of valid test cases, +/// whose lifetime is at least that of the test suite (i.e., static). +/// +/// # Examples +/// +/// ```ignore +/// extern "C" fn test_fn(_test: *mut kernel::bindings::kunit) { +/// let actual = 1 + 1; +/// let expected = 2; +/// assert_eq!(actual, expected); +/// } +/// +/// static mut KUNIT_TEST_CASES: [kernel::bindings::kunit_case; 2] = [ +/// kernel::kunit::kunit_case(c"name", test_fn), +/// pin_init::zeroed(), +/// ]; +/// kernel::kunit_unsafe_test_suite!(suite_name, KUNIT_TEST_CASES); +/// ``` +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_unsafe_test_suite { + ($name:ident, $test_cases:ident) => { + const _: () = { + const KUNIT_TEST_SUITE_NAME: [::kernel::ffi::c_char; 256] = { + let name_u8 = ::core::stringify!($name).as_bytes(); + let mut ret = [0; 256]; + + if name_u8.len() > 255 { + panic!(concat!( + "The test suite name `", + ::core::stringify!($name), + "` exceeds the maximum length of 255 bytes." + )); + } + + let mut i = 0; + while i < name_u8.len() { + ret[i] = name_u8[i] as ::kernel::ffi::c_char; + i += 1; + } + + ret + }; + + static mut KUNIT_TEST_SUITE: ::kernel::bindings::kunit_suite = + ::kernel::bindings::kunit_suite { + name: KUNIT_TEST_SUITE_NAME, + #[allow(unused_unsafe)] + // SAFETY: `$test_cases` is passed in by the user, and + // (as documented) must be valid for the lifetime of + // the suite (i.e., static). + test_cases: unsafe { + ::core::ptr::addr_of_mut!($test_cases) + .cast::<::kernel::bindings::kunit_case>() + }, + suite_init: None, + suite_exit: None, + init: None, + exit: None, + attr: ::kernel::bindings::kunit_attributes { + speed: ::kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + status_comment: [0; 256usize], + debugfs: ::core::ptr::null_mut(), + log: ::core::ptr::null_mut(), + suite_init_err: 0, + is_init: false, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + }; + + #[used(compiler)] + #[allow(unused_unsafe)] + #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")] + static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite = + // SAFETY: `KUNIT_TEST_SUITE` is static. + unsafe { ::core::ptr::addr_of_mut!(KUNIT_TEST_SUITE) }; + }; + }; +} + +/// Returns whether we are currently running a KUnit test. +/// +/// In some cases, you need to call test-only code from outside the test case, for example, to +/// create a function mock. This function allows to change behavior depending on whether we are +/// currently running a KUnit test or not. +/// +/// # Examples +/// +/// This example shows how a function can be mocked to return a well-known value while testing: +/// +/// ``` +/// # use kernel::kunit::in_kunit_test; +/// fn fn_mock_example(n: i32) -> i32 { +/// if in_kunit_test() { +/// return 100; +/// } +/// +/// n + 1 +/// } +/// +/// let mock_res = fn_mock_example(5); +/// assert_eq!(mock_res, 100); +/// ``` +pub fn in_kunit_test() -> bool { + // SAFETY: `kunit_get_current_test()` is always safe to call (it has fallbacks for + // when KUnit is not enabled). + !unsafe { bindings::kunit_get_current_test() }.is_null() +} + +#[cfg(CONFIG_RUST_KUNIT_SELFTEST)] +#[kunit_tests(rust_kernel_kunit)] +mod tests { + use super::*; + + #[test] + fn rust_test_kunit_example_test() { + assert_eq!(1 + 1, 2); + } + + #[test] + fn rust_test_kunit_in_kunit_test() { + assert!(in_kunit_test()); + } + + #[test] + #[cfg(not(all()))] + fn rust_test_kunit_always_disabled_test() { + // This test should never run because of the `cfg`. + assert!(false); + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs new file mode 100644 index 000000000..4d5c96ddc --- /dev/null +++ b/rust/kernel/lib.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` crate. +//! +//! This crate contains the kernel APIs that have been ported or wrapped for +//! usage by Rust code in the kernel and is shared by all of them. +//! +//! In other words, all the rest of the Rust code in the kernel (e.g. kernel +//! modules written in Rust) depends on [`core`] and this crate. +//! +//! If you need a kernel C API that is not ported or wrapped yet here, then +//! do so first instead of bypassing this crate. + +#![no_std] +// +// Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on +// the unstable features in use. +// +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// +// Stable since Rust 1.89.0. +#![feature(generic_arg_infer)] +// +// Expected to become stable. +#![feature(arbitrary_self_types)] +#![feature(derive_coerce_pointee)] +// +// To be determined. +#![feature(used_with_arg)] +// +// `feature(file_with_nul)` is stable since Rust 1.92.0. Before Rust 1.89.0, it did not exist, so +// enable it conditionally. +#![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))] + +// Ensure conditional compilation based on the kernel configuration works; +// otherwise we may silently break things like initcall handling. +#[cfg(not(CONFIG_RUST))] +compile_error!("Missing kernel configuration for conditional compilation"); + +// Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). +extern crate self as kernel; + +pub use ffi; + +pub mod acpi; +pub mod alloc; +#[cfg(CONFIG_AUXILIARY_BUS)] +pub mod auxiliary; +pub mod bitfield; +pub mod bitmap; +pub mod bits; +#[cfg(CONFIG_BLOCK)] +pub mod block; +pub mod bug; +pub mod build_assert; +pub mod clk; +#[cfg(CONFIG_CONFIGFS_FS)] +pub mod configfs; +pub mod cpu; +#[cfg(CONFIG_CPU_FREQ)] +pub mod cpufreq; +pub mod cpumask; +pub mod cred; +pub mod debugfs; +pub mod device; +pub mod device_id; +pub mod devres; +pub mod dma; +pub mod driver; +#[cfg(CONFIG_DRM = "y")] +pub mod drm; +pub mod error; +pub mod faux; +#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] +pub mod firmware; +pub mod fmt; +pub mod fs; +#[cfg(CONFIG_RUST_FWCTL_ABSTRACTIONS)] +pub mod fwctl; +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod gpu; +#[cfg(CONFIG_I2C = "y")] +pub mod i2c; +pub mod id_pool; +#[doc(hidden)] +pub mod impl_flags; +pub mod init; +pub mod interop; +pub mod interrupt; +pub mod io; +pub mod ioctl; +pub mod iommu; +pub mod iov; +pub mod irq; +pub mod jump_label; +#[cfg(CONFIG_KUNIT)] +pub mod kunit; +pub mod list; +pub mod maple_tree; +pub mod miscdevice; +pub mod mm; +pub mod module; +pub mod module_param; +#[cfg(CONFIG_NET)] +pub mod net; +pub mod num; +pub mod of; +#[cfg(CONFIG_PM_OPP)] +pub mod opp; +pub mod page; +#[cfg(CONFIG_PCI)] +pub mod pci; +pub mod pid_namespace; +pub mod platform; +pub mod prelude; +pub mod print; +pub mod processor; +pub mod ptr; +#[cfg(CONFIG_RUST_PWM_ABSTRACTIONS)] +pub mod pwm; +pub mod rbtree; +pub mod regulator; +pub mod revocable; +pub mod safety; +pub mod scatterlist; +pub mod security; +pub mod seq_file; +#[cfg(CONFIG_RUST_SERIAL_DEV_BUS_ABSTRACTIONS)] +pub mod serdev; +pub mod sizes; +#[cfg(CONFIG_SOC_BUS)] +pub mod soc; +#[doc(hidden)] +pub mod std_vendor; +pub mod str; +pub mod sync; +pub mod task; +pub mod time; +pub mod tracepoint; +pub mod transmute; +pub mod types; +pub mod uaccess; +#[cfg(CONFIG_USB = "y")] +pub mod usb; +pub mod workqueue; +pub mod xarray; + +#[doc(hidden)] +pub use bindings; +pub use macros; +pub use module::{ + InPlaceModule, + Module, + ModuleMetadata, + ThisModule, // +}; +pub use uapi; + +/// Prefix to appear before log messages printed from within the `kernel` crate. +const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; + +/// Dummy module type for `#[vtable]` `impl` blocks within the `kernel` crate (e.g. KUnit tests). +// The `allow` is needed since it may be unused (e.g. KUnit tests may be disabled). +#[allow(dead_code)] +struct LocalModule; + +impl ModuleMetadata for LocalModule { + const NAME: &'static str::CStr = c"rust_kernel"; + + const THIS_MODULE: ThisModule = { + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe { ThisModule::from_ptr(core::ptr::null_mut()) } + }; +} + +#[cfg(not(testlib))] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + pr_emerg!("{}\n", info); + // SAFETY: FFI call. + unsafe { bindings::BUG() }; +} + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or +/// [`Opaque::cast_from`] to resolve the mismatch. +/// +/// [`Opaque`]: crate::types::Opaque +/// [`Opaque::cast_into`]: crate::types::Opaque::cast_into +/// [`Opaque::cast_from`]: crate::types::Opaque::cast_from +/// +/// # Safety +/// +/// The pointer passed to this macro, and the pointer returned by this macro, must both be in +/// bounds of the same allocation. +/// +/// # Examples +/// +/// ``` +/// # use kernel::container_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr: *const _ = &test.b; +/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be +/// // in-bounds of the same allocation as `b_ptr`. +/// let test_alias = unsafe { container_of!(b_ptr, Test, b) }; +/// assert!(core::ptr::eq(&test, test_alias)); +/// ``` +#[macro_export] +macro_rules! container_of { + ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{ + let offset: usize = ::core::mem::offset_of!($Container, $($fields)*); + let field_ptr = $field_ptr; + let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>(); + $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut()); + container_ptr + }} +} + +/// Helper for [`container_of!`]. +#[doc(hidden)] +pub fn assert_same_type(_: T, _: T) {} + +/// Helper for `.rs.S` files. +#[doc(hidden)] +#[macro_export] +macro_rules! concat_literals { + ($( $asm:literal )* ) => { + ::core::concat!($($asm),*) + }; +} + +/// Wrapper around `asm!` configured for use in the kernel. +/// +/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!` +/// syntax. +// For x86, `asm!` uses intel syntax by default, but we want to use at&t syntax in the kernel. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[macro_export] +macro_rules! asm { + ($($asm:expr),* ; $($rest:tt)*) => { + ::core::arch::asm!( $($asm)*, options(att_syntax), $($rest)* ) + }; +} + +/// Wrapper around `asm!` configured for use in the kernel. +/// +/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!` +/// syntax. +// For non-x86 arches we just pass through to `asm!`. +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] +#[macro_export] +macro_rules! asm { + ($($asm:expr),* ; $($rest:tt)*) => { + ::core::arch::asm!( $($asm)*, $($rest)* ) + }; +} + +/// Gets the C string file name of a [`Location`]. +/// +/// If `Location::file_as_c_str()` is not available, returns a string that warns about it. +/// +/// [`Location`]: core::panic::Location +/// +/// # Examples +/// +/// ``` +/// # use kernel::file_from_location; +/// +/// #[track_caller] +/// fn foo() { +/// let caller = core::panic::Location::caller(); +/// +/// // Output: +/// // - A path like "rust/kernel/example.rs" if `file_as_c_str()` is available. +/// // - "" otherwise. +/// let caller_file = file_from_location(caller); +/// +/// // Prints out the message with caller's file name. +/// pr_info!("foo() called in file {caller_file:?}\n"); +/// +/// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) { +/// # assert_eq!(Ok(caller.file()), caller_file.to_str()); +/// # } +/// } +/// +/// # foo(); +/// ``` +#[inline] +pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr { + #[cfg(CONFIG_RUSTC_HAS_FILE_AS_C_STR)] + { + loc.file_as_c_str() + } + + #[cfg(all(CONFIG_RUSTC_HAS_FILE_WITH_NUL, not(CONFIG_RUSTC_HAS_FILE_AS_C_STR)))] + { + loc.file_with_nul() + } + + #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))] + { + let _ = loc; + c"" + } +} diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs new file mode 100644 index 000000000..0f367264e --- /dev/null +++ b/rust/kernel/list.rs @@ -0,0 +1,1222 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A linked list implementation. + +use crate::sync::ArcBorrow; +use crate::types::Opaque; +use core::iter::{DoubleEndedIterator, FusedIterator}; +use core::marker::PhantomData; +use core::ptr; +use pin_init::PinInit; + +mod impl_list_item_mod; +#[doc(inline)] +pub use self::impl_list_item_mod::{ + impl_has_list_links, + impl_has_list_links_self_ptr, + impl_list_item, + HasListLinks, + HasSelfPtr, // +}; + +mod arc; +#[doc(inline)] +pub use self::arc::{ + impl_list_arc_safe, + AtomicTracker, + ListArc, + ListArcSafe, + TryNewListArc, // +}; + +mod arc_field; +#[doc(inline)] +pub use self::arc_field::{ + define_list_arc_field_getter, + ListArcField, // +}; + +/// A linked list. +/// +/// All elements in this linked list will be [`ListArc`] references to the value. Since a value can +/// only have one `ListArc` (for each pair of prev/next pointers), this ensures that the same +/// prev/next pointers are not used for several linked lists. +/// +/// # Invariants +/// +/// * If the list is empty, then `first` is null. Otherwise, `first` points at the `ListLinks` +/// field of the first element in the list. +/// * All prev/next pointers in `ListLinks` fields of items in the list are valid and form a cycle. +/// * For every item in the list, the list owns the associated [`ListArc`] reference and has +/// exclusive access to the `ListLinks` field. +/// +/// # Examples +/// +/// Use [`ListLinks`] as the type of the intrusive field. +/// +/// ``` +/// use kernel::list::*; +/// +/// #[pin_data] +/// struct BasicItem { +/// value: i32, +/// #[pin] +/// links: ListLinks, +/// } +/// +/// impl BasicItem { +/// fn new(value: i32) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinks::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// impl_list_arc_safe! { +/// impl ListArcSafe<0> for BasicItem { untracked; } +/// } +/// impl_list_item! { +/// impl ListItem<0> for BasicItem { using ListLinks { self.links }; } +/// } +/// +/// // Create a new empty list. +/// let mut list = List::new(); +/// { +/// assert!(list.is_empty()); +/// } +/// +/// // Insert 3 elements using `push_back()`. +/// list.push_back(BasicItem::new(15)?); +/// list.push_back(BasicItem::new(10)?); +/// list.push_back(BasicItem::new(30)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [15, 10, 30] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 10); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 15); +/// } +/// +/// // Insert 3 elements using `push_front()`. +/// list.push_front(BasicItem::new(15)?); +/// list.push_front(BasicItem::new(10)?); +/// list.push_front(BasicItem::new(30)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [30, 10, 15] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_front()` and verify the content. +/// { +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 10); +/// } +/// +/// // Push `list2` to `list` through `push_all_back()`. +/// // list: [15] +/// // list2: [25, 35] +/// { +/// let mut list2 = List::new(); +/// list2.push_back(BasicItem::new(25)?); +/// list2.push_back(BasicItem::new(35)?); +/// +/// list.push_all_back(&mut list2); +/// +/// // list: [15, 25, 35] +/// // list2: [] +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 25); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 35); +/// assert!(iter.next().is_none()); +/// assert!(list2.is_empty()); +/// } +/// # Result::<(), Error>::Ok(()) +/// ``` +/// +/// Use [`ListLinksSelfPtr`] as the type of the intrusive field. This allows a list of trait object +/// type. +/// +/// ``` +/// use kernel::list::*; +/// +/// trait Foo { +/// fn foo(&self) -> (&'static str, i32); +/// } +/// +/// #[pin_data] +/// struct DTWrap { +/// #[pin] +/// links: ListLinksSelfPtr>, +/// value: T, +/// } +/// +/// impl DTWrap { +/// fn new(value: T) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinksSelfPtr::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// impl_list_arc_safe! { +/// impl{T: ?Sized} ListArcSafe<0> for DTWrap { untracked; } +/// } +/// impl_list_item! { +/// impl ListItem<0> for DTWrap { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// // Create a new empty list. +/// let mut list = List::>::new(); +/// { +/// assert!(list.is_empty()); +/// } +/// +/// struct A(i32); +/// // `A` returns the inner value for `foo`. +/// impl Foo for A { fn foo(&self) -> (&'static str, i32) { ("a", self.0) } } +/// +/// struct B; +/// // `B` always returns 42. +/// impl Foo for B { fn foo(&self) -> (&'static str, i32) { ("b", 42) } } +/// +/// // Insert 3 element using `push_back()`. +/// list.push_back(DTWrap::new(A(15))?); +/// list.push_back(DTWrap::new(A(32))?); +/// list.push_back(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [A(15), A(32), B] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// } +/// +/// // Insert 3 elements using `push_front()`. +/// list.push_front(DTWrap::new(A(15))?); +/// list.push_front(DTWrap::new(A(32))?); +/// list.push_front(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [B, A(32), A(15)] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// } +/// +/// // Push `list2` to `list` through `push_all_back()`. +/// // list: [B] +/// // list2: [B, A(25)] +/// { +/// let mut list2 = List::>::new(); +/// list2.push_back(DTWrap::new(B)?); +/// list2.push_back(DTWrap::new(A(25))?); +/// +/// list.push_all_back(&mut list2); +/// +/// // list: [B, B, A(25)] +/// // list2: [] +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 25)); +/// assert!(iter.next().is_none()); +/// assert!(list2.is_empty()); +/// } +/// # Result::<(), Error>::Ok(()) +/// ``` +pub struct List, const ID: u64 = 0> { + first: *mut ListLinksFields, + _ty: PhantomData>, +} + +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Send for List +where + ListArc: Send, + T: ?Sized + ListItem, +{ +} +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Sync for List +where + ListArc: Sync, + T: ?Sized + ListItem, +{ +} + +/// Implemented by types where a [`ListArc`] can be inserted into a [`List`]. +/// +/// # Safety +/// +/// Implementers must ensure that they provide the guarantees documented on methods provided by +/// this trait. +/// +/// [`ListArc`]: ListArc +pub unsafe trait ListItem: ListArcSafe { + /// Views the [`ListLinks`] for this value. + /// + /// # Guarantees + /// + /// If there is a previous call to `prepare_to_insert` and there is no call to `post_remove` + /// since the most recent such call, then this returns the same pointer as the one returned by + /// the most recent call to `prepare_to_insert`. + /// + /// Otherwise, the returned pointer points at a read-only [`ListLinks`] with two null pointers. + /// + /// # Safety + /// + /// The provided pointer must point at a valid value. (It need not be in an `Arc`.) + unsafe fn view_links(me: *const Self) -> *mut ListLinks; + + /// View the full value given its [`ListLinks`] field. + /// + /// Can only be used when the value is in a list. + /// + /// # Guarantees + /// + /// * Returns the same pointer as the one passed to the most recent call to `prepare_to_insert`. + /// * The returned pointer is valid until the next call to `post_remove`. + /// + /// # Safety + /// + /// * The provided pointer must originate from the most recent call to `prepare_to_insert`, or + /// from a call to `view_links` that happened after the most recent call to + /// `prepare_to_insert`. + /// * Since the most recent call to `prepare_to_insert`, the `post_remove` method must not have + /// been called. + unsafe fn view_value(me: *mut ListLinks) -> *const Self; + + /// This is called when an item is inserted into a [`List`]. + /// + /// # Guarantees + /// + /// The caller is granted exclusive access to the returned [`ListLinks`] until `post_remove` is + /// called. + /// + /// # Safety + /// + /// * The provided pointer must point at a valid value in an [`Arc`]. + /// * Calls to `prepare_to_insert` and `post_remove` on the same value must alternate. + /// * The caller must own the [`ListArc`] for this value. + /// * The caller must not give up ownership of the [`ListArc`] unless `post_remove` has been + /// called after this call to `prepare_to_insert`. + /// + /// [`Arc`]: crate::sync::Arc + unsafe fn prepare_to_insert(me: *const Self) -> *mut ListLinks; + + /// This undoes a previous call to `prepare_to_insert`. + /// + /// # Guarantees + /// + /// The returned pointer is the pointer that was originally passed to `prepare_to_insert`. + /// + /// # Safety + /// + /// The provided pointer must be the pointer returned by the most recent call to + /// `prepare_to_insert`. + unsafe fn post_remove(me: *mut ListLinks) -> *const Self; +} + +#[repr(C)] +#[derive(Copy, Clone)] +struct ListLinksFields { + next: *mut ListLinksFields, + prev: *mut ListLinksFields, +} + +/// The prev/next pointers for an item in a linked list. +/// +/// # Invariants +/// +/// The fields are null if and only if this item is not in a list. +#[repr(transparent)] +pub struct ListLinks { + // This type is `!Unpin` for aliasing reasons as the pointers are part of an intrusive linked + // list. + inner: Opaque, +} + +// SAFETY: The only way to access/modify the pointers inside of `ListLinks` is via holding the +// associated `ListArc`. Since that type correctly implements `Send`, it is impossible to +// move this an instance of this type to a different thread if the pointees are `!Send`. +unsafe impl Send for ListLinks {} +// SAFETY: The type is opaque so immutable references to a ListLinks are useless. Therefore, it's +// okay to have immutable access to a ListLinks from several threads at once. +unsafe impl Sync for ListLinks {} + +impl ListLinks { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + } + } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn fields(me: *mut Self) -> *mut ListLinksFields { + // SAFETY: The caller promises that the pointer is valid. + unsafe { Opaque::cast_into(ptr::addr_of!((*me).inner)) } + } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn from_fields(me: *mut ListLinksFields) -> *mut Self { + me.cast() + } +} + +/// Similar to [`ListLinks`], but also contains a pointer to the full value. +/// +/// This type can be used instead of [`ListLinks`] to support lists with trait objects. +#[repr(C)] +pub struct ListLinksSelfPtr { + /// The `ListLinks` field inside this value. + /// + /// This is public so that it can be used with `impl_has_list_links!`. + pub inner: ListLinks, + // UnsafeCell is not enough here because we use `Opaque::uninit` as a dummy value, and + // `ptr::null()` doesn't work for `T: ?Sized`. + self_ptr: Opaque<*const T>, +} + +// SAFETY: The fields of a ListLinksSelfPtr can be moved across thread boundaries. +unsafe impl Send for ListLinksSelfPtr {} +// SAFETY: The type is opaque so immutable references to a ListLinksSelfPtr are useless. Therefore, +// it's okay to have immutable access to a ListLinks from several threads at once. +// +// Note that `inner` being a public field does not prevent this type from being opaque, since +// `inner` is a opaque type. +unsafe impl Sync for ListLinksSelfPtr {} + +impl ListLinksSelfPtr { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + }, + self_ptr: Opaque::uninit(), + } + } + + /// Returns a pointer to the self pointer. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + pub unsafe fn raw_get_self_ptr(me: *const Self) -> *const Opaque<*const T> { + // SAFETY: The caller promises that the pointer is valid. + unsafe { ptr::addr_of!((*me).self_ptr) } + } +} + +impl, const ID: u64> List { + /// Creates a new empty list. + pub const fn new() -> Self { + Self { + first: ptr::null_mut(), + _ty: PhantomData, + } + } + + /// Returns whether this list is empty. + pub fn is_empty(&self) -> bool { + self.first.is_null() + } + + /// Inserts `item` before `next` in the cycle. + /// + /// Returns a pointer to the newly inserted element. Never changes `self.first` unless the list + /// is empty. + /// + /// # Safety + /// + /// * `next` must be an element in this list or null. + /// * if `next` is null, then the list must be empty. + unsafe fn insert_inner( + &mut self, + item: ListArc, + next: *mut ListLinksFields, + ) -> *mut ListLinksFields { + let raw_item = ListArc::into_raw(item); + // SAFETY: + // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. + // * Since we have ownership of the `ListArc`, `post_remove` must have been called after + // the most recent call to `prepare_to_insert`, if any. + // * We own the `ListArc`. + // * Removing items from this list is always done using `remove_internal_inner`, which + // calls `post_remove` before giving up ownership. + let list_links = unsafe { T::prepare_to_insert(raw_item) }; + // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. + let item = unsafe { ListLinks::fields(list_links) }; + + // Check if the list is empty. + if next.is_null() { + // SAFETY: The caller just gave us ownership of these fields. + // INVARIANT: A linked list with one item should be cyclic. + unsafe { + (*item).next = item; + (*item).prev = item; + } + self.first = item; + } else { + // SAFETY: By the type invariant, this pointer is valid or null. We just checked that + // it's not null, so it must be valid. + let prev = unsafe { (*next).prev }; + // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us + // ownership of the fields on `item`. + // INVARIANT: This correctly inserts `item` between `prev` and `next`. + unsafe { + (*item).next = next; + (*item).prev = prev; + (*prev).next = item; + (*next).prev = item; + } + } + + item + } + + /// Add the provided item to the back of the list. + pub fn push_back(&mut self, item: ListArc) { + // SAFETY: + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + unsafe { self.insert_inner(item, self.first) }; + } + + /// Add the provided item to the front of the list. + pub fn push_front(&mut self, item: ListArc) { + // SAFETY: + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + let new_elem = unsafe { self.insert_inner(item, self.first) }; + + // INVARIANT: `new_elem` is in the list because we just inserted it. + self.first = new_elem; + } + + /// Removes the last item from this list. + pub fn pop_back(&mut self) -> Option> { + if self.is_empty() { + return None; + } + + // SAFETY: We just checked that the list is not empty. + let last = unsafe { (*self.first).prev }; + // SAFETY: The last item of this list is in this list. + Some(unsafe { self.remove_internal(last) }) + } + + /// Removes the first item from this list. + pub fn pop_front(&mut self) -> Option> { + if self.is_empty() { + return None; + } + + // SAFETY: The first item of this list is in this list. + Some(unsafe { self.remove_internal(self.first) }) + } + + /// Removes the provided item from this list and returns it. + /// + /// This returns `None` if the item is not in the list. (Note that by the safety requirements, + /// this means that the item is not in any list.) + /// + /// When using this method, be careful with using `mem::take` on the same list as that may + /// result in violating the safety requirements of this method. + /// + /// # Safety + /// + /// `item` must not be in a different linked list (with the same id). + pub unsafe fn remove(&mut self, item: &T) -> Option> { + // SAFETY: TODO. + let mut item = unsafe { ListLinks::fields(T::view_links(item)) }; + // SAFETY: The user provided a reference, and reference are never dangling. + // + // As for why this is not a data race, there are two cases: + // + // * If `item` is not in any list, then these fields are read-only and null. + // * If `item` is in this list, then we have exclusive access to these fields since we + // have a mutable reference to the list. + // + // In either case, there's no race. + let ListLinksFields { next, prev } = unsafe { *item }; + + debug_assert_eq!(next.is_null(), prev.is_null()); + if !next.is_null() { + // This is really a no-op, but this ensures that `item` is a raw pointer that was + // obtained without going through a pointer->reference->pointer conversion roundtrip. + // This ensures that the list is valid under the more restrictive strict provenance + // ruleset. + // + // SAFETY: We just checked that `next` is not null, and it's not dangling by the + // list invariants. + unsafe { + debug_assert_eq!(item, (*next).prev); + item = (*next).prev; + } + + // SAFETY: We just checked that `item` is in a list, so the caller guarantees that it + // is in this list. The pointers are in the right order. + Some(unsafe { self.remove_internal_inner(item, next, prev) }) + } else { + None + } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// `item` must point at an item in this list. + unsafe fn remove_internal(&mut self, item: *mut ListLinksFields) -> ListArc { + // SAFETY: The caller promises that this pointer is not dangling, and there's no data race + // since we have a mutable reference to the list containing `item`. + let ListLinksFields { next, prev } = unsafe { *item }; + // SAFETY: The pointers are ok and in the right order. + unsafe { self.remove_internal_inner(item, next, prev) } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// The `item` pointer must point at an item in this list, and we must have `(*item).next == + /// next` and `(*item).prev == prev`. + unsafe fn remove_internal_inner( + &mut self, + item: *mut ListLinksFields, + next: *mut ListLinksFields, + prev: *mut ListLinksFields, + ) -> ListArc { + // SAFETY: We have exclusive access to the pointers of items in the list, and the prev/next + // pointers are always valid for items in a list. + // + // INVARIANT: There are three cases: + // * If the list has at least three items, then after removing the item, `prev` and `next` + // will be next to each other. + // * If the list has two items, then the remaining item will point at itself. + // * If the list has one item, then `next == prev == item`, so these writes have no + // effect. The list remains unchanged and `item` is still in the list for now. + unsafe { + (*next).prev = prev; + (*prev).next = next; + } + // SAFETY: We have exclusive access to items in the list. + // INVARIANT: `item` is being removed, so the pointers should be null. + unsafe { + (*item).prev = ptr::null_mut(); + (*item).next = ptr::null_mut(); + } + // INVARIANT: There are three cases: + // * If `item` was not the first item, then `self.first` should remain unchanged. + // * If `item` was the first item and there is another item, then we just updated + // `prev->next` to `next`, which is the new first item, and setting `item->next` to null + // did not modify `prev->next`. + // * If `item` was the only item in the list, then `prev == item`, and we just set + // `item->next` to null, so this correctly sets `first` to null now that the list is + // empty. + if self.first == item { + // SAFETY: The `prev` pointer is the value that `item->prev` had when it was in this + // list, so it must be valid. There is no race since `prev` is still in the list and we + // still have exclusive access to the list. + self.first = unsafe { (*prev).next }; + } + + // SAFETY: `item` used to be in the list, so it is dereferenceable by the type invariants + // of `List`. + let list_links = unsafe { ListLinks::from_fields(item) }; + // SAFETY: Any pointer in the list originates from a `prepare_to_insert` call. + let raw_item = unsafe { T::post_remove(list_links) }; + // SAFETY: The above call to `post_remove` guarantees that we can recreate the `ListArc`. + unsafe { ListArc::from_raw(raw_item) } + } + + /// Moves all items from `other` into `self`. + /// + /// The items of `other` are added to the back of `self`, so the last item of `other` becomes + /// the last item of `self`. + pub fn push_all_back(&mut self, other: &mut List) { + // First, we insert the elements into `self`. At the end, we make `other` empty. + if self.is_empty() { + // INVARIANT: All of the elements in `other` become elements of `self`. + self.first = other.first; + } else if !other.is_empty() { + let other_first = other.first; + // SAFETY: The other list is not empty, so this pointer is valid. + let other_last = unsafe { (*other_first).prev }; + let self_first = self.first; + // SAFETY: The self list is not empty, so this pointer is valid. + let self_last = unsafe { (*self_first).prev }; + + // SAFETY: We have exclusive access to both lists, so we can update the pointers. + // INVARIANT: This correctly sets the pointers to merge both lists. We do not need to + // update `self.first` because the first element of `self` does not change. + unsafe { + (*self_first).prev = other_last; + (*other_last).next = self_first; + (*self_last).next = other_first; + (*other_first).prev = self_last; + } + } + + // INVARIANT: The other list is now empty, so update its pointer. + other.first = ptr::null_mut(); + } + + /// Returns a cursor that points before the first element of the list. + pub fn cursor_front(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `self.first` is in this list. + Cursor { + next: self.first, + list: self, + } + } + + /// Returns a cursor that points after the last element in the list. + pub fn cursor_back(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `next` is allowed to be null. + Cursor { + next: core::ptr::null_mut(), + list: self, + } + } + + /// Creates an iterator over the list. + pub fn iter(&self) -> Iter<'_, T, ID> { + // INVARIANT: If the list is empty, both pointers are null. Otherwise, both pointers point + // at the first element of the same list. + Iter { + current: self.first, + stop: self.first, + _ty: PhantomData, + } + } +} + +impl, const ID: u64> Default for List { + fn default() -> Self { + List::new() + } +} + +impl, const ID: u64> Drop for List { + fn drop(&mut self) { + while let Some(item) = self.pop_front() { + drop(item); + } + } +} + +/// An iterator over a [`List`]. +/// +/// # Invariants +/// +/// * There must be a [`List`] that is immutably borrowed for the duration of `'a`. +/// * The `current` pointer is null or points at a value in that [`List`]. +/// * The `stop` pointer is equal to the `first` field of that [`List`]. +#[derive(Clone)] +pub struct Iter<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + current: *mut ListLinksFields, + stop: *mut ListLinksFields, + _ty: PhantomData<&'a ListArc>, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Iterator for Iter<'a, T, ID> { + type Item = ArcBorrow<'a, T>; + + fn next(&mut self) -> Option> { + if self.current.is_null() { + return None; + } + + let current = self.current; + + // SAFETY: We just checked that `current` is not null, so it is in a list, and hence not + // dangling. There's no race because the iterator holds an immutable borrow to the list. + let next = unsafe { (*current).next }; + // INVARIANT: If `current` was the last element of the list, then this updates it to null. + // Otherwise, we update it to the next element. + self.current = if next != self.stop { + next + } else { + ptr::null_mut() + }; + + // SAFETY: The `current` pointer points at a value in the list. + let item = unsafe { T::view_value(ListLinks::from_fields(current)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the list. However, the `ArcBorrow` is annotated with the iterator's + // lifetime, and the list is immutably borrowed for that lifetime. + // * Values in a list never have a `UniqueArc` reference. + Some(unsafe { ArcBorrow::from_raw(item) }) + } +} + +/// A cursor into a [`List`]. +/// +/// A cursor always rests between two elements in the list. This means that a cursor has a previous +/// and next element, but no current element. It also means that it's possible to have a cursor +/// into an empty list. +/// +/// # Examples +/// +/// ``` +/// use kernel::prelude::*; +/// use kernel::list::{List, ListArc, ListLinks}; +/// +/// #[pin_data] +/// struct ListItem { +/// value: u32, +/// #[pin] +/// links: ListLinks, +/// } +/// +/// impl ListItem { +/// fn new(value: u32) -> Result> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinks::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for ListItem { untracked; } +/// } +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for ListItem { using ListLinks { self.links }; } +/// } +/// +/// // Use a cursor to remove the first element with the given value. +/// fn remove_first(list: &mut List, value: u32) -> Option> { +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// return Some(next.remove()); +/// } +/// cursor.move_next(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove the last element with the given value. +/// fn remove_last(list: &mut List, value: u32) -> Option> { +/// let mut cursor = list.cursor_back(); +/// while let Some(prev) = cursor.peek_prev() { +/// if prev.value == value { +/// return Some(prev.remove()); +/// } +/// cursor.move_prev(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove all elements with the given value. The removed elements are moved to +/// // a new list. +/// fn remove_all(list: &mut List, value: u32) -> List { +/// let mut out = List::new(); +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// out.push_back(next.remove()); +/// } else { +/// cursor.move_next(); +/// } +/// } +/// out +/// } +/// +/// // Use a cursor to insert a value at a specific index. Returns an error if the index is out of +/// // bounds. +/// fn insert_at(list: &mut List, new: ListArc, idx: usize) -> Result { +/// let mut cursor = list.cursor_front(); +/// for _ in 0..idx { +/// if !cursor.move_next() { +/// return Err(EINVAL); +/// } +/// } +/// cursor.insert_next(new); +/// Ok(()) +/// } +/// +/// // Merge two sorted lists into a single sorted list. +/// fn merge_sorted(list: &mut List, merge: List) { +/// let mut cursor = list.cursor_front(); +/// for to_insert in merge { +/// while let Some(next) = cursor.peek_next() { +/// if to_insert.value < next.value { +/// break; +/// } +/// cursor.move_next(); +/// } +/// cursor.insert_prev(to_insert); +/// } +/// } +/// +/// let mut list = List::new(); +/// list.push_back(ListItem::new(14)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(10)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(15)?); +/// list.push_back(ListItem::new(14)?); +/// assert_eq!(remove_all(&mut list, 12).iter().count(), 2); +/// // [14, 10, 15, 14] +/// assert!(remove_first(&mut list, 14).is_some()); +/// // [10, 15, 14] +/// insert_at(&mut list, ListItem::new(12)?, 2)?; +/// // [10, 15, 12, 14] +/// assert!(remove_last(&mut list, 15).is_some()); +/// // [10, 12, 14] +/// +/// let mut list2 = List::new(); +/// list2.push_back(ListItem::new(11)?); +/// list2.push_back(ListItem::new(13)?); +/// merge_sorted(&mut list, list2); +/// +/// let mut items = list.into_iter(); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 11); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 12); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 13); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 14); +/// assert!(items.next().is_none()); +/// # Result::<(), Error>::Ok(()) +/// ``` +/// +/// # Invariants +/// +/// The `next` pointer is null or points a value in `list`. +pub struct Cursor<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + list: &'a mut List, + /// Points at the element after this cursor, or null if the cursor is after the last element. + next: *mut ListLinksFields, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Cursor<'a, T, ID> { + /// Returns a pointer to the element before the cursor. + /// + /// Returns null if there is no element before the cursor. + fn prev_ptr(&self) -> *mut ListLinksFields { + let mut next = self.next; + let first = self.list.first; + if next == first { + // We are before the first element. + return core::ptr::null_mut(); + } + + if next.is_null() { + // We are after the last element, so we need a pointer to the last element, which is + // the same as `(*first).prev`. + next = first; + } + + // SAFETY: `next` can't be null, because then `first` must also be null, but in that case + // we would have exited at the `next == first` check. Thus, `next` is an element in the + // list, so we can access its `prev` pointer. + unsafe { (*next).prev } + } + + /// Access the element after this cursor. + pub fn peek_next(&mut self) -> Option> { + if self.next.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `self.next` is non-null, so it must be in `self.list`. + // * `ptr` is equal to `self.next`. + Some(CursorPeek { + ptr: self.next, + cursor: self, + }) + } + + /// Access the element before this cursor. + pub fn peek_prev(&mut self) -> Option> { + let prev = self.prev_ptr(); + + if prev.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `prev` is non-null, so it must be in `self.list`. + // * `self.prev_ptr()` never returns `self.next`. + Some(CursorPeek { + ptr: prev, + cursor: self, + }) + } + + /// Move the cursor one element forward. + /// + /// If the cursor is after the last element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_next(&mut self) -> bool { + if self.next.is_null() { + return false; + } + + // SAFETY: `self.next` is an element in the list and we borrow the list mutably, so we can + // access the `next` field. + let mut next = unsafe { (*self.next).next }; + + if next == self.list.first { + next = core::ptr::null_mut(); + } + + // INVARIANT: `next` is either null or the next element after an element in the list. + self.next = next; + true + } + + /// Move the cursor one element backwards. + /// + /// If the cursor is before the first element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_prev(&mut self) -> bool { + if self.next == self.list.first { + return false; + } + + // INVARIANT: `prev_ptr()` always returns a pointer that is null or in the list. + self.next = self.prev_ptr(); + true + } + + /// Inserts an element where the cursor is pointing and get a pointer to the new element. + fn insert_inner(&mut self, item: ListArc) -> *mut ListLinksFields { + let ptr = if self.next.is_null() { + self.list.first + } else { + self.next + }; + // SAFETY: + // * `ptr` is an element in the list or null. + // * if `ptr` is null, then `self.list.first` is null so the list is empty. + let item = unsafe { self.list.insert_inner(item, ptr) }; + if self.next == self.list.first { + // INVARIANT: We just inserted `item`, so it's a member of list. + self.list.first = item; + } + item + } + + /// Insert an element at this cursor's location. + pub fn insert(mut self, item: ListArc) { + // This is identical to `insert_prev`, but consumes the cursor. This is helpful because it + // reduces confusion when the last operation on the cursor is an insertion; in that case, + // you just want to insert the element at the cursor, and it is confusing that the call + // involves the word prev or next. + self.insert_inner(item); + } + + /// Inserts an element after this cursor. + /// + /// After insertion, the new element will be after the cursor. + pub fn insert_next(&mut self, item: ListArc) { + self.next = self.insert_inner(item); + } + + /// Inserts an element before this cursor. + /// + /// After insertion, the new element will be before the cursor. + pub fn insert_prev(&mut self, item: ListArc) { + self.insert_inner(item); + } + + /// Remove the next element from the list. + pub fn remove_next(&mut self) -> Option> { + self.peek_next().map(|v| v.remove()) + } + + /// Remove the previous element from the list. + pub fn remove_prev(&mut self) -> Option> { + self.peek_prev().map(|v| v.remove()) + } +} + +/// References the element in the list next to the cursor. +/// +/// # Invariants +/// +/// * `ptr` is an element in `self.cursor.list`. +/// * `ISNEXT == (self.ptr == self.cursor.next)`. +pub struct CursorPeek<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> { + cursor: &'a mut Cursor<'b, T, ID>, + ptr: *mut ListLinksFields, +} + +impl<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> + CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + /// Remove the element from the list. + pub fn remove(self) -> ListArc { + if ISNEXT { + self.cursor.move_next(); + } + + // INVARIANT: `self.ptr` is not equal to `self.cursor.next` due to the above `move_next` + // call. + // SAFETY: By the type invariants of `Self`, `next` is not null, so `next` is an element of + // `self.cursor.list` by the type invariants of `Cursor`. + unsafe { self.cursor.list.remove_internal(self.ptr) } + } + + /// Access this value as an [`ArcBorrow`]. + pub fn arc(&self) -> ArcBorrow<'_, T> { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `ArcBorrow` holds + // an immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable + // access requires first releasing the immutable borrow on the `CursorPeek`. + // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc` + // reference, and `UniqueArc` references must be unique. + unsafe { ArcBorrow::from_raw(me) } + } +} + +impl<'a, 'b, T: ?Sized + ListItem, const ISNEXT: bool, const ID: u64> core::ops::Deref + for CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + // If you change the `ptr` field to have type `ArcBorrow<'a, T>`, it might seem like you could + // get rid of the `CursorPeek::arc` method and change the deref target to `ArcBorrow<'a, T>`. + // However, that doesn't work because 'a is too long. You could obtain an `ArcBorrow<'a, T>` + // and then call `CursorPeek::remove` without giving up the `ArcBorrow<'a, T>`, which would be + // unsound. + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + + // SAFETY: The value cannot be removed from the list for the duration of the lifetime + // annotated on the returned `&T`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `&T` holds an + // immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable access + // requires first releasing the immutable borrow on the `CursorPeek`. + unsafe { &*me } + } +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> FusedIterator for Iter<'a, T, ID> {} + +impl<'a, T: ?Sized + ListItem, const ID: u64> IntoIterator for &'a List { + type IntoIter = Iter<'a, T, ID>; + type Item = ArcBorrow<'a, T>; + + fn into_iter(self) -> Iter<'a, T, ID> { + self.iter() + } +} + +/// An owning iterator into a [`List`]. +pub struct IntoIter, const ID: u64 = 0> { + list: List, +} + +impl, const ID: u64> Iterator for IntoIter { + type Item = ListArc; + + fn next(&mut self) -> Option> { + self.list.pop_front() + } +} + +impl, const ID: u64> FusedIterator for IntoIter {} + +impl, const ID: u64> DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option> { + self.list.pop_back() + } +} + +impl, const ID: u64> IntoIterator for List { + type IntoIter = IntoIter; + type Item = ListArc; + + fn into_iter(self) -> IntoIter { + IntoIter { list: self } + } +} diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs new file mode 100644 index 000000000..209b1173c --- /dev/null +++ b/rust/kernel/list/arc.rs @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A wrapper around `Arc` for linked lists. + +use crate::alloc::{AllocError, Flags}; +use crate::prelude::*; +use crate::sync::atomic::{ordering, AtomicFlag}; +use crate::sync::{Arc, ArcBorrow, UniqueArc}; +use core::marker::PhantomPinned; +use core::ops::Deref; +use core::pin::Pin; + +/// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for +/// this id. +/// +/// Types that implement this trait should include some kind of logic for keeping track of whether +/// a [`ListArc`] exists or not. We refer to this logic as "the tracking inside `T`". +/// +/// We allow the case where the tracking inside `T` thinks that a [`ListArc`] exists, but actually, +/// there isn't a [`ListArc`]. However, we do not allow the opposite situation where a [`ListArc`] +/// exists, but the tracking thinks it doesn't. This is because the former can at most result in us +/// failing to create a [`ListArc`] when the operation could succeed, whereas the latter can result +/// in the creation of two [`ListArc`] references. Only the latter situation can lead to memory +/// safety issues. +/// +/// A consequence of the above is that you may implement the tracking inside `T` by not actually +/// keeping track of anything. To do this, you always claim that a [`ListArc`] exists, even if +/// there isn't one. This implementation is allowed by the above rule, but it means that +/// [`ListArc`] references can only be created if you have ownership of *all* references to the +/// refcounted object, as you otherwise have no way of knowing whether a [`ListArc`] exists. +pub trait ListArcSafe { + /// Informs the tracking inside this type that it now has a [`ListArc`] reference. + /// + /// This method may be called even if the tracking inside this type thinks that a `ListArc` + /// reference exists. (But only if that's not actually the case.) + /// + /// # Safety + /// + /// Must not be called if a [`ListArc`] already exist for this value. + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>); + + /// Informs the tracking inside this type that there is no [`ListArc`] reference anymore. + /// + /// # Safety + /// + /// Must only be called if there is no [`ListArc`] reference, but the tracking thinks there is. + unsafe fn on_drop_list_arc(&self); +} + +/// Declares that this type is able to safely attempt to create `ListArc`s at any time. +/// +/// # Safety +/// +/// The guarantees of `try_new_list_arc` must be upheld. +pub unsafe trait TryNewListArc: ListArcSafe { + /// Attempts to convert an `Arc` into an `ListArc`. Returns `true` if the + /// conversion was successful. + /// + /// This method should not be called directly. Use [`ListArc::try_from_arc`] instead. + /// + /// # Guarantees + /// + /// If this call returns `true`, then there is no [`ListArc`] pointing to this value. + /// Additionally, this call will have transitioned the tracking inside `Self` from not thinking + /// that a [`ListArc`] exists, to thinking that a [`ListArc`] exists. + fn try_new_list_arc(&self) -> bool; +} + +/// Declares that this type supports [`ListArc`]. +/// +/// This macro supports a few different strategies for implementing the tracking inside the type: +/// +/// * The `untracked` strategy does not actually keep track of whether a [`ListArc`] exists. When +/// using this strategy, the only way to create a [`ListArc`] is using a [`UniqueArc`]. +/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user must specify +/// which field to defer the tracking to. The field must implement [`ListArcSafe`]. If the field +/// implements [`TryNewListArc`], then the type will also implement [`TryNewListArc`]. +/// +/// The `tracked_by` strategy is usually used by deferring to a field of type +/// [`AtomicTracker`]. However, it is also possible to defer the tracking to another struct +/// using also using this macro. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_list_arc_safe { + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {} + unsafe fn on_drop_list_arc(&self) {} + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { + tracked_by $field:ident : $fty:ty; + } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) { + ::pin_init::assert_pinned!($t, $field, $fty, inline); + + // SAFETY: This field is structurally pinned as per the above assertion. + let field = unsafe { + ::core::pin::Pin::map_unchecked_mut(self, |me| &mut me.$field) + }; + // SAFETY: The caller promises that there is no `ListArc`. + unsafe { + <$fty as $crate::list::ListArcSafe<$num>>::on_create_list_arc_from_unique(field) + }; + } + unsafe fn on_drop_list_arc(&self) { + // SAFETY: The caller promises that there is no `ListArc` reference, and also + // promises that the tracking thinks there is a `ListArc` reference. + unsafe { <$fty as $crate::list::ListArcSafe<$num>>::on_drop_list_arc(&self.$field) }; + } + } + unsafe impl$(<$($generics)*>)? $crate::list::TryNewListArc<$num> for $t + where + $fty: TryNewListArc<$num>, + { + fn try_new_list_arc(&self) -> bool { + <$fty as $crate::list::TryNewListArc<$num>>::try_new_list_arc(&self.$field) + } + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + + () => {}; +} +pub use impl_list_arc_safe; + +/// A wrapper around [`Arc`] that's guaranteed unique for the given id. +/// +/// The `ListArc` type can be thought of as a special reference to a refcounted object that owns the +/// permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring +/// that each object has only one `ListArc` reference, the owner of that reference is assured +/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a [`List`], +/// the [`List`] takes ownership of the `ListArc` reference. +/// +/// There are various strategies to ensuring that a value has only one `ListArc` reference. The +/// simplest is to convert a [`UniqueArc`] into a `ListArc`. However, the refcounted object could +/// also keep track of whether a `ListArc` exists using a boolean, which could allow for the +/// creation of new `ListArc` references from an [`Arc`] reference. Whatever strategy is used, the +/// relevant tracking is referred to as "the tracking inside `T`", and the [`ListArcSafe`] trait +/// (and its subtraits) are used to update the tracking when a `ListArc` is created or destroyed. +/// +/// Note that we allow the case where the tracking inside `T` thinks that a `ListArc` exists, but +/// actually, there isn't a `ListArc`. However, we do not allow the opposite situation where a +/// `ListArc` exists, but the tracking thinks it doesn't. This is because the former can at most +/// result in us failing to create a `ListArc` when the operation could succeed, whereas the latter +/// can result in the creation of two `ListArc` references. +/// +/// While this `ListArc` is unique for the given id, there still might exist normal `Arc` +/// references to the object. +/// +/// # Invariants +/// +/// * Each reference counted object has at most one `ListArc` for each value of `ID`. +/// * The tracking inside `T` is aware that a `ListArc` reference exists. +/// +/// [`List`]: crate::list::List +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct ListArc +where + T: ListArcSafe + ?Sized, +{ + arc: Arc, +} + +impl, const ID: u64> ListArc { + /// Constructs a new reference counted instance of `T`. + #[inline] + pub fn new(contents: T, flags: Flags) -> Result { + Ok(Self::from(UniqueArc::new(contents, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + // We don't implement `InPlaceInit` because `ListArc` is implicitly pinned. This is similar to + // what we do for `Arc`. + #[inline] + pub fn pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_pin_init(init, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// This is equivalent to [`ListArc::pin_init`], since a [`ListArc`] is always pinned. + #[inline] + pub fn init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_init(init, flags)?)) + } +} + +impl From> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(unique: UniqueArc) -> Self { + Self::from(Pin::from(unique)) + } +} + +impl From>> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a pinned [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(mut unique: Pin>) -> Self { + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { T::on_create_list_arc_from_unique(unique.as_mut()) }; + let arc = Arc::from(unique); + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc`, + // so we can create a `ListArc`. + unsafe { Self::transmute_from_arc(arc) } + } +} + +impl ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Creates two `ListArc`s from a [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_unique(unique: UniqueArc) -> (Self, ListArc) + where + T: ListArcSafe, + { + Self::pair_from_pin_unique(Pin::from(unique)) + } + + /// Creates two `ListArc`s from a pinned [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_pin_unique( + mut unique: Pin>, + ) -> (Self, ListArc) + where + T: ListArcSafe, + { + build_assert!(ID != ID2); + + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + + let arc1 = Arc::from(unique); + let arc2 = Arc::clone(&arc1); + + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc` + // for both IDs (which are different), so we can create two `ListArc`s. + unsafe { + ( + Self::transmute_from_arc(arc1), + ListArc::transmute_from_arc(arc2), + ) + } + } + + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc(arc: Arc) -> Result> + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Ok(unsafe { Self::transmute_from_arc(arc) }) + } else { + Err(arc) + } + } + + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc_borrow(arc: ArcBorrow<'_, T>) -> Option + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Some(unsafe { Self::transmute_from_arc(Arc::from(arc)) }) + } else { + None + } + } + + /// Try to create a new `ListArc`. + /// + /// If it's not possible to create a new `ListArc`, then the `Arc` is dropped. This will never + /// run the destructor of the value. + pub fn try_from_arc_or_drop(arc: Arc) -> Option + where + T: TryNewListArc, + { + match Self::try_from_arc(arc) { + Ok(list_arc) => Some(list_arc), + Err(arc) => Arc::into_unique_or_drop(arc).map(Self::from), + } + } + + /// Transmutes an [`Arc`] into a `ListArc` without updating the tracking inside `T`. + /// + /// # Safety + /// + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + unsafe fn transmute_from_arc(arc: Arc) -> Self { + // INVARIANT: By the safety requirements, the invariants on `ListArc` are satisfied. + Self { arc } + } + + /// Transmutes a `ListArc` into an [`Arc`] without updating the tracking inside `T`. + /// + /// After this call, the tracking inside `T` will still think that there is a `ListArc` + /// reference. + #[inline] + fn transmute_to_arc(self) -> Arc { + // Use a transmute to skip destructor. + // + // SAFETY: ListArc is repr(transparent). + unsafe { core::mem::transmute(self) } + } + + /// Convert ownership of this `ListArc` into a raw pointer. + /// + /// The returned pointer is indistinguishable from pointers returned by [`Arc::into_raw`]. The + /// tracking inside `T` will still think that a `ListArc` exists after this call. + #[inline] + pub fn into_raw(self) -> *const T { + Arc::into_raw(Self::transmute_to_arc(self)) + } + + /// Take ownership of the `ListArc` from a raw pointer. + /// + /// # Safety + /// + /// * `ptr` must satisfy the safety requirements of [`Arc::from_raw`]. + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The pointer satisfies the safety requirements for `Arc::from_raw`. + let arc = unsafe { Arc::from_raw(ptr) }; + // SAFETY: The value doesn't already have a `ListArc` reference, but the tracking thinks it + // does. + unsafe { Self::transmute_from_arc(arc) } + } + + /// Converts the `ListArc` into an [`Arc`]. + #[inline] + pub fn into_arc(self) -> Arc { + let arc = Self::transmute_to_arc(self); + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is. + unsafe { T::on_drop_list_arc(&arc) }; + arc + } + + /// Clone a `ListArc` into an [`Arc`]. + #[inline] + pub fn clone_arc(&self) -> Arc { + self.arc.clone() + } + + /// Returns a reference to an [`Arc`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`&Arc`] (e.g., in a method + /// receiver), but we have a [`ListArc`] instead. + /// + /// [`&Arc`]: Arc + #[inline] + pub fn as_arc(&self) -> &Arc { + &self.arc + } + + /// Returns an [`ArcBorrow`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method + /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. + #[inline] + pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { + self.arc.as_arc_borrow() + } + + /// Compare whether two [`ListArc`] pointers reference the same underlying object. + #[inline] + pub fn ptr_eq(this: &Self, other: &Self) -> bool { + Arc::ptr_eq(&this.arc, &other.arc) + } +} + +impl Deref for ListArc +where + T: ListArcSafe + ?Sized, +{ + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.arc.deref() + } +} + +impl Drop for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is by the type + // invariants on `Self`. + unsafe { T::on_drop_list_arc(&self.arc) }; + } +} + +impl AsRef> for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn as_ref(&self) -> &Arc { + self.as_arc() + } +} + +/// A utility for tracking whether a [`ListArc`] exists using an atomic. +/// +/// # Invariants +/// +/// If the boolean is `false`, then there is no [`ListArc`] for this value. +#[repr(transparent)] +pub struct AtomicTracker { + inner: AtomicFlag, + // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. + _pin: PhantomPinned, +} + +impl AtomicTracker { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: AtomicFlag::new(false), + _pin: PhantomPinned, + } + } + + fn project_inner(self: Pin<&mut Self>) -> &mut AtomicFlag { + // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable + // reference to it even if we only have a pinned reference to `self`. + unsafe { &mut Pin::into_inner_unchecked(self).inner } + } +} + +impl ListArcSafe for AtomicTracker { + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>) { + // INVARIANT: We just created a ListArc, so the boolean should be true. + *self.project_inner().get_mut() = true; + } + + unsafe fn on_drop_list_arc(&self) { + // INVARIANT: We just dropped a ListArc, so the boolean should be false. + self.inner.store(false, ordering::Release); + } +} + +// SAFETY: If this method returns `true`, then by the type invariant there is no `ListArc` before +// this call, so it is okay to create a new `ListArc`. +// +// The acquire ordering will synchronize with the release store from the destruction of any +// previous `ListArc`, so if there was a previous `ListArc`, then the destruction of the previous +// `ListArc` happens-before the creation of the new `ListArc`. +unsafe impl TryNewListArc for AtomicTracker { + fn try_new_list_arc(&self) -> bool { + // INVARIANT: If this method returns true, then the boolean used to be false, and is no + // longer false, so it is okay for the caller to create a new [`ListArc`]. + self.inner.cmpxchg(false, true, ordering::Acquire).is_ok() + } +} diff --git a/rust/kernel/list/arc_field.rs b/rust/kernel/list/arc_field.rs new file mode 100644 index 000000000..2ad8aea55 --- /dev/null +++ b/rust/kernel/list/arc_field.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A field that is exclusively owned by a [`ListArc`]. +//! +//! This can be used to have reference counted struct where one of the reference counted pointers +//! has exclusive access to a field of the struct. +//! +//! [`ListArc`]: crate::list::ListArc + +use core::cell::UnsafeCell; + +/// A field owned by a specific [`ListArc`]. +/// +/// [`ListArc`]: crate::list::ListArc +pub struct ListArcField { + value: UnsafeCell, +} + +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Send for ListArcField {} +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Sync for ListArcField {} + +impl ListArcField { + /// Creates a new `ListArcField`. + pub fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + } + } + + /// Access the value when we have exclusive access to the `ListArcField`. + /// + /// This allows access to the field using an `UniqueArc` instead of a `ListArc`. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } + + /// Unsafely assert that you have shared access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have shared access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + pub unsafe fn assert_ref(&self) -> &T { + // SAFETY: The caller has shared access to the `ListArc`, so they also have shared access + // to this field. + unsafe { &*self.value.get() } + } + + /// Unsafely assert that you have mutable access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have mutable access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + #[expect(clippy::mut_from_ref)] + pub unsafe fn assert_mut(&self) -> &mut T { + // SAFETY: The caller has exclusive access to the `ListArc`, so they also have exclusive + // access to this field. + unsafe { &mut *self.value.get() } + } +} + +/// Defines getters for a [`ListArcField`]. +#[macro_export] +#[doc(hidden)] +macro_rules! define_list_arc_field_getter { + ($pub:vis fn $name:ident(&self $(<$id:tt>)?) -> &$typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a $crate::list::ListArc) -> &'a $typ { + let field = &(&**self).$field; + // SAFETY: We have a shared reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_ref(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + ($pub:vis fn $name:ident(&mut self $(<$id:tt>)?) -> &mut $typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a mut $crate::list::ListArc) -> &'a mut $typ { + let field = &(&**self).$field; + // SAFETY: We have a mutable reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_mut(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + () => {}; +} +pub use define_list_arc_field_getter; diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs new file mode 100644 index 000000000..5a3eac9f3 --- /dev/null +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Helpers for implementing list traits safely. + +/// Declares that this type has a [`ListLinks`] field. +/// +/// This trait is only used to help implement [`ListItem`] safely. If [`ListItem`] is implemented +/// manually, then this trait is not needed. Use the [`impl_has_list_links!`] macro to implement +/// this trait. +/// +/// # Safety +/// +/// The methods on this trait must have exactly the behavior that the definitions given below have. +/// +/// [`ListLinks`]: crate::list::ListLinks +/// [`ListItem`]: crate::list::ListItem +pub unsafe trait HasListLinks { + /// Returns a pointer to the [`ListLinks`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + /// + /// [`ListLinks`]: crate::list::ListLinks + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut crate::list::ListLinks; +} + +/// Implements the [`HasListLinks`] trait for the given type. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_has_list_links { + ($(impl$({$($generics:tt)*})? + HasListLinks$(<$id:tt>)? + for $self:ty + { self$(.$field:ident)* } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + // Statically ensure that `$(.field)*` doesn't follow any pointers. + // + // Cannot be `const` because `$self` may contain generics and E0401 says constants + // "can't use {`Self`,generic parameters} from outer item". + if false { let _: usize = ::core::mem::offset_of!(Self, $($field).*); } + + // SAFETY: The caller promises that the pointer is not dangling. We know that this + // expression doesn't follow any pointers, as the `offset_of!` invocation above + // would otherwise not compile. + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) } + } + } + )*}; +} +pub use impl_has_list_links; + +/// Declares that the [`ListLinks`] field in this struct is inside a +/// [`ListLinksSelfPtr`]. +/// +/// # Safety +/// +/// The [`ListLinks`] field of this struct at [`HasListLinks::raw_get_list_links`] must be +/// inside a [`ListLinksSelfPtr`]. +/// +/// [`ListLinks`]: crate::list::ListLinks +/// [`ListLinksSelfPtr`]: crate::list::ListLinksSelfPtr +pub unsafe trait HasSelfPtr +where + Self: HasListLinks, +{ +} + +/// Implements the [`HasListLinks`] and [`HasSelfPtr`] traits for the given type. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_has_list_links_self_ptr { + ($(impl$({$($generics:tt)*})? + HasSelfPtr<$item_type:ty $(, $id:tt)?> + for $self:ty + { self$(.$field:ident)* } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + + // SAFETY: TODO. + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; + ptr.cast() + } + } + )*}; +} +pub use impl_has_list_links_self_ptr; + +/// Implements the [`ListItem`] trait for the given type. +/// +/// Requires that the type implements [`HasListLinks`]. Use the [`impl_has_list_links!`] macro to +/// implement that trait. +/// +/// [`ListItem`]: crate::list::ListItem +/// +/// # Examples +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinks, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinks { self.links }; } +/// } +/// +/// struct ListLinksHolder { +/// inner: kernel::list::ListLinks, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem { +/// value: Result, +/// #[pin] +/// links: ListLinksHolder, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem { using ListLinks { self.links.inner }; } +/// } +/// ``` +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinksSelfPtr, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// struct ListLinksSelfPtrHolder { +/// inner: kernel::list::ListLinksSelfPtr>, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem { +/// value: Result, +/// #[pin] +/// links: ListLinksSelfPtrHolder, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem { +/// using ListLinksSelfPtr { self.links.inner }; +/// } +/// } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! impl_list_item { + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinks { self$(.$field:ident)* }; + })* + ) => {$( + $crate::list::impl_has_list_links! { + impl$({$($generics)*})? HasListLinks<$num> for $self { self$(.$field)* } + } + + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // is implemented in terms of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller guarantees that `me` points at a valid value of type `Self`. + unsafe { + >::raw_get_list_links(me.cast_mut()) + } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + unsafe { $crate::container_of!(me, Self, $($field).*) } + } + + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value. + unsafe { >::view_links(me) } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + unsafe { $crate::container_of!(me, Self, $($field).*) } + } + } + )*}; + + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinksSelfPtr { self$(.$field:ident)* }; + })* + ) => {$( + $crate::list::impl_has_list_links_self_ptr! { + impl$({$($generics)*})? HasSelfPtr<$self> for $self { self$(.$field)* } + } + + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + let links_field = unsafe { >::view_links(me) }; + + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); + + // SAFETY: This value is not accessed in any other places than `prepare_to_insert`, + // `post_remove`, or `view_value`. By the safety requirements of those methods, + // none of these three methods may be called in parallel with this call to + // `prepare_to_insert`, so this write will not race with any other access to the + // value. + unsafe { ::core::ptr::write(cell_inner, me) }; + + links_field + } + + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // returns the return value of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + unsafe { + >::raw_get_list_links(me.cast_mut()) + } + } + + // This function is also used as the implementation of `post_remove`, so the caller + // may choose to satisfy the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + // + // GUARANTEES: (always) + // * This returns the same pointer as the one passed to the most recent call to + // `prepare_to_insert` since that call wrote that pointer to this location. The value + // is only modified in `prepare_to_insert`, so it has not been modified since the + // most recent call. + // + // GUARANTEES: (only when using the `view_value` safety requirements) + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); + + // SAFETY: This is not a data race, because the only function that writes to this + // value is `prepare_to_insert`, but by the safety requirements the + // `prepare_to_insert` method may not be called in parallel with `view_value` or + // `post_remove`. + unsafe { ::core::ptr::read(cell_inner) } + } + + // GUARANTEES: + // The first guarantee of `view_value` is exactly what `post_remove` guarantees. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: This specific implementation of `view_value` allows the caller to + // promise the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + unsafe { >::view_value(me) } + } + } + )*}; +} +pub use impl_list_item; diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs new file mode 100644 index 000000000..265d6396a --- /dev/null +++ b/rust/kernel/maple_tree.rs @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Maple trees. +//! +//! C header: [`include/linux/maple_tree.h`](srctree/include/linux/maple_tree.h) +//! +//! Reference: + +use core::{ + marker::PhantomData, + ops::{Bound, RangeBounds}, + ptr, +}; + +use kernel::{ + alloc::Flags, + error::to_result, + prelude::*, + types::{ForeignOwnable, Opaque}, +}; + +/// A maple tree optimized for storing non-overlapping ranges. +/// +/// # Invariants +/// +/// Each range in the maple tree owns an instance of `T`. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct MapleTree { + #[pin] + tree: Opaque, + _p: PhantomData, +} + +/// A maple tree with `MT_FLAGS_ALLOC_RANGE` set. +/// +/// All methods on [`MapleTree`] are also accessible on this type. +#[pin_data] +#[repr(transparent)] +pub struct MapleTreeAlloc { + #[pin] + tree: MapleTree, +} + +// Make MapleTree methods usable on MapleTreeAlloc. +impl core::ops::Deref for MapleTreeAlloc { + type Target = MapleTree; + + #[inline] + fn deref(&self) -> &MapleTree { + &self.tree + } +} + +#[inline] +fn to_maple_range(range: impl RangeBounds) -> Option<(usize, usize)> { + let first = match range.start_bound() { + Bound::Included(start) => *start, + Bound::Excluded(start) => start.checked_add(1)?, + Bound::Unbounded => 0, + }; + + let last = match range.end_bound() { + Bound::Included(end) => *end, + Bound::Excluded(end) => end.checked_sub(1)?, + Bound::Unbounded => usize::MAX, + }; + + if last < first { + return None; + } + + Some((first, last)) +} + +impl MapleTree { + /// Create a new maple tree. + /// + /// The tree will use the regular implementation with a higher branching factor, rather than + /// the allocation tree. + #[inline] + pub fn new() -> impl PinInit { + pin_init!(MapleTree { + // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be + // destroyed in Drop before the memory location becomes invalid. + tree <- Opaque::ffi_init(|slot| unsafe { bindings::mt_init_flags(slot, 0) }), + _p: PhantomData, + }) + } + + /// Insert the value at the given index. + /// + /// # Errors + /// + /// If the maple tree already contains a range using the given index, then this call will + /// return an [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{InsertErrorKind, MapleTree}; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let the_answer = KBox::new(42, GFP_KERNEL)?; + /// + /// // These calls will succeed. + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(101, twenty, GFP_KERNEL)?; + /// + /// // This will fail because the index is already in use. + /// assert_eq!( + /// tree.insert(100, the_answer, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::Occupied, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn insert(&self, index: usize, value: T, gfp: Flags) -> Result<(), InsertError> { + self.insert_range(index..=index, value, gfp) + } + + /// Insert a value to the specified range, failing on overlap. + /// + /// This accepts the usual types of Rust ranges using the `..` and `..=` syntax for exclusive + /// and inclusive ranges respectively. The range must not be empty, and must not overlap with + /// any existing range. + /// + /// # Errors + /// + /// If the maple tree already contains an overlapping range, then this call will return an + /// [`InsertErrorKind::Occupied`]. It may also fail if memory allocation fails or if the + /// requested range is invalid (e.g. empty). + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{InsertErrorKind, MapleTree}; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let the_answer = KBox::new(42, GFP_KERNEL)?; + /// let hundred = KBox::new(100, GFP_KERNEL)?; + /// + /// // Insert the value 10 at the indices 100 to 499. + /// tree.insert_range(100..500, ten, GFP_KERNEL)?; + /// + /// // Insert the value 20 at the indices 500 to 1000. + /// tree.insert_range(500..=1000, twenty, GFP_KERNEL)?; + /// + /// // This will fail due to overlap with the previous range on index 1000. + /// assert_eq!( + /// tree.insert_range(1000..1200, the_answer, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::Occupied, + /// ); + /// + /// // When using .. to specify the range, you must be careful to ensure that the range is + /// // non-empty. + /// assert_eq!( + /// tree.insert_range(72..72, hundred, GFP_KERNEL).unwrap_err().cause, + /// InsertErrorKind::InvalidRequest, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + pub fn insert_range(&self, range: R, value: T, gfp: Flags) -> Result<(), InsertError> + where + R: RangeBounds, + { + let Some((first, last)) = to_maple_range(range) else { + return Err(InsertError { + value, + cause: InsertErrorKind::InvalidRequest, + }); + }; + + let ptr = T::into_foreign(value); + + // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`. + let res = to_result(unsafe { + bindings::mtree_insert_range(self.tree.get(), first, last, ptr, gfp.as_raw()) + }); + + if let Err(err) = res { + // SAFETY: As `mtree_insert_range` failed, it is safe to take back ownership. + let value = unsafe { T::from_foreign(ptr) }; + + let cause = if err == ENOMEM { + InsertErrorKind::AllocError(kernel::alloc::AllocError) + } else if err == EEXIST { + InsertErrorKind::Occupied + } else { + InsertErrorKind::InvalidRequest + }; + Err(InsertError { value, cause }) + } else { + Ok(()) + } + } + + /// Erase the range containing the given index. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// + /// tree.insert_range(100..500, ten, GFP_KERNEL)?; + /// tree.insert(67, twenty, GFP_KERNEL)?; + /// + /// assert_eq!(tree.erase(67).map(|v| *v), Some(20)); + /// assert_eq!(tree.erase(275).map(|v| *v), Some(10)); + /// + /// // The previous call erased the entire range, not just index 275. + /// assert!(tree.erase(127).is_none()); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn erase(&self, index: usize) -> Option { + // SAFETY: `self.tree` contains a valid maple tree. + let ret = unsafe { bindings::mtree_erase(self.tree.get(), index) }; + + // SAFETY: If the pointer is not null, then we took ownership of a valid instance of `T` + // from the tree. + unsafe { T::try_from_foreign(ret) } + } + + /// Lock the internal spinlock. + #[inline] + pub fn lock(&self) -> MapleGuard<'_, T> { + // SAFETY: It's safe to lock the spinlock in a maple tree. + unsafe { bindings::spin_lock(self.ma_lock()) }; + + // INVARIANT: We just took the spinlock. + MapleGuard(self) + } + + #[inline] + fn ma_lock(&self) -> *mut bindings::spinlock_t { + // SAFETY: This pointer offset operation stays in-bounds. + let lock_ptr = unsafe { &raw mut (*self.tree.get()).__bindgen_anon_1.ma_lock }; + lock_ptr.cast() + } + + /// Free all `T` instances in this tree. + /// + /// # Safety + /// + /// This frees Rust data referenced by the maple tree without removing it from the maple tree, + /// leaving it in an invalid state. The caller must ensure that this invalid state cannot be + /// observed by the end-user. + unsafe fn free_all_entries(self: Pin<&mut Self>) { + // SAFETY: The caller provides exclusive access to the entire maple tree, so we have + // exclusive access to the entire maple tree despite not holding the lock. + let mut ma_state = unsafe { MaState::new_raw(self.into_ref().get_ref(), 0, usize::MAX) }; + + loop { + // This uses the raw accessor because we're destroying pointers without removing them + // from the maple tree, which is only valid because this is the destructor. + // + // Take the rcu lock because mas_find_raw() requires that you hold either the spinlock + // or the rcu read lock. This is only really required if memory reclaim might + // reallocate entries in the tree, as we otherwise have exclusive access. That feature + // doesn't exist yet, so for now, taking the rcu lock only serves the purpose of + // silencing lockdep. + let ptr = { + let _rcu = kernel::sync::rcu::Guard::new(); + ma_state.mas_find_raw(usize::MAX) + }; + if ptr.is_null() { + break; + } + // SAFETY: By the type invariants, this pointer references a valid value of type `T`. + // By the safety requirements, it is okay to free it without removing it from the maple + // tree. + drop(unsafe { T::from_foreign(ptr) }); + } + } +} + +#[pinned_drop] +impl PinnedDrop for MapleTree { + #[inline] + fn drop(mut self: Pin<&mut Self>) { + // We only iterate the tree if the Rust value has a destructor. + if core::mem::needs_drop::() { + // SAFETY: Other than the below `mtree_destroy` call, the tree will not be accessed + // after this call. + unsafe { self.as_mut().free_all_entries() }; + } + + // SAFETY: The tree is valid, and will not be accessed after this call. + unsafe { bindings::mtree_destroy(self.tree.get()) }; + } +} + +/// A reference to a [`MapleTree`] that owns the inner lock. +/// +/// # Invariants +/// +/// This guard owns the inner spinlock. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree); + +impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, we hold this spinlock. + unsafe { bindings::spin_unlock(self.0.ma_lock()) }; + } +} + +impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { + /// Create a [`MaState`] protected by this lock guard. + pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { + // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s + // read/write permissions to the maple tree. + unsafe { MaState::new_raw(self.0, first, end) } + } + + /// Load the value at the given index. + /// + /// # Examples + /// + /// Read the value while holding the spinlock. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// let mut lock = tree.lock(); + /// assert_eq!(lock.load(100).map(|v| *v), Some(10)); + /// assert_eq!(lock.load(200).map(|v| *v), Some(20)); + /// assert_eq!(lock.load(300).map(|v| *v), None); + /// # Ok::<_, Error>(()) + /// ``` + /// + /// Increment refcount under the lock, to keep value alive afterwards. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// use kernel::sync::Arc; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = Arc::new(10, GFP_KERNEL)?; + /// let twenty = Arc::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// // Briefly take the lock to increment the refcount. + /// let value = tree.lock().load(100).map(Arc::from); + /// + /// // At this point, another thread might remove the value. + /// tree.erase(100); + /// + /// // But we can still access it because we took a refcount. + /// assert_eq!(value.map(|v| *v), Some(10)); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn load(&mut self, index: usize) -> Option> { + // SAFETY: `self.tree` contains a valid maple tree. + let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) }; + if ret.is_null() { + return None; + } + + // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It is + // safe to borrow the instance mutably because the signature of this function enforces that + // the mutable borrow is not used after the spinlock is dropped. + Some(unsafe { T::borrow_mut(ret) }) + } +} + +impl MapleTreeAlloc { + /// Create a new allocation tree. + pub fn new() -> impl PinInit { + let tree = pin_init!(MapleTree { + // SAFETY: This initializes a maple tree into a pinned slot. The maple tree will be + // destroyed in Drop before the memory location becomes invalid. + tree <- Opaque::ffi_init(|slot| unsafe { + bindings::mt_init_flags(slot, bindings::MT_FLAGS_ALLOC_RANGE) + }), + _p: PhantomData, + }); + + pin_init!(MapleTreeAlloc { tree <- tree }) + } + + /// Insert an entry with the given size somewhere in the given range. + /// + /// The maple tree will search for a location in the given range where there is space to insert + /// the new range. If there is not enough available space, then an error will be returned. + /// + /// The index of the new range is returned. + /// + /// # Examples + /// + /// ``` + /// use kernel::maple_tree::{MapleTreeAlloc, AllocErrorKind}; + /// + /// let tree = KBox::pin_init(MapleTreeAlloc::>::new(), GFP_KERNEL)?; + /// + /// let ten = KBox::new(10, GFP_KERNEL)?; + /// let twenty = KBox::new(20, GFP_KERNEL)?; + /// let thirty = KBox::new(30, GFP_KERNEL)?; + /// let hundred = KBox::new(100, GFP_KERNEL)?; + /// + /// // Allocate three ranges. + /// let idx1 = tree.alloc_range(100, ten, ..1000, GFP_KERNEL)?; + /// let idx2 = tree.alloc_range(100, twenty, ..1000, GFP_KERNEL)?; + /// let idx3 = tree.alloc_range(100, thirty, ..1000, GFP_KERNEL)?; + /// + /// assert_eq!(idx1, 0); + /// assert_eq!(idx2, 100); + /// assert_eq!(idx3, 200); + /// + /// // This will fail because the remaining space is too small. + /// assert_eq!( + /// tree.alloc_range(800, hundred, ..1000, GFP_KERNEL).unwrap_err().cause, + /// AllocErrorKind::Busy, + /// ); + /// # Ok::<_, Error>(()) + /// ``` + pub fn alloc_range( + &self, + size: usize, + value: T, + range: R, + gfp: Flags, + ) -> Result> + where + R: RangeBounds, + { + let Some((min, max)) = to_maple_range(range) else { + return Err(AllocError { + value, + cause: AllocErrorKind::InvalidRequest, + }); + }; + + let ptr = T::into_foreign(value); + let mut index = 0; + + // SAFETY: The tree is valid, and we are passing a pointer to an owned instance of `T`. + let res = to_result(unsafe { + bindings::mtree_alloc_range( + self.tree.tree.get(), + &mut index, + ptr, + size, + min, + max, + gfp.as_raw(), + ) + }); + + if let Err(err) = res { + // SAFETY: As `mtree_alloc_range` failed, it is safe to take back ownership. + let value = unsafe { T::from_foreign(ptr) }; + + let cause = if err == ENOMEM { + AllocErrorKind::AllocError(kernel::alloc::AllocError) + } else if err == EBUSY { + AllocErrorKind::Busy + } else { + AllocErrorKind::InvalidRequest + }; + Err(AllocError { value, cause }) + } else { + Ok(index) + } + } +} + +/// A helper type used for navigating a [`MapleTree`]. +/// +/// # Invariants +/// +/// For the duration of `'tree`: +/// +/// * The `ma_state` references a valid `MapleTree`. +/// * The `ma_state` has read/write access to the tree. +pub struct MaState<'tree, T: ForeignOwnable> { + state: bindings::ma_state, + _phantom: PhantomData<&'tree mut MapleTree>, +} + +impl<'tree, T: ForeignOwnable> MaState<'tree, T> { + /// Initialize a new `MaState` with the given tree. + /// + /// # Safety + /// + /// The caller must ensure that this `MaState` has read/write access to the maple tree. + #[inline] + unsafe fn new_raw(mt: &'tree MapleTree, first: usize, end: usize) -> Self { + // INVARIANT: + // * Having a reference ensures that the `MapleTree` is valid for `'tree`. + // * The caller ensures that we have read/write access. + Self { + state: bindings::ma_state { + tree: mt.tree.get(), + index: first, + last: end, + node: ptr::null_mut(), + status: bindings::maple_status_ma_start, + min: 0, + max: usize::MAX, + alloc: ptr::null_mut(), + mas_flags: 0, + store_type: bindings::store_type_wr_invalid, + ..Default::default() + }, + _phantom: PhantomData, + } + } + + #[inline] + fn as_raw(&mut self) -> *mut bindings::ma_state { + &raw mut self.state + } + + #[inline] + fn mas_find_raw(&mut self, max: usize) -> *mut c_void { + // SAFETY: By the type invariants, the `ma_state` is active and we have read/write access + // to the tree. + unsafe { bindings::mas_find(self.as_raw(), max) } + } + + /// Find the next entry in the maple tree. + /// + /// # Examples + /// + /// Iterate the maple tree. + /// + /// ``` + /// use kernel::maple_tree::MapleTree; + /// use kernel::sync::Arc; + /// + /// let tree = KBox::pin_init(MapleTree::>::new(), GFP_KERNEL)?; + /// + /// let ten = Arc::new(10, GFP_KERNEL)?; + /// let twenty = Arc::new(20, GFP_KERNEL)?; + /// tree.insert(100, ten, GFP_KERNEL)?; + /// tree.insert(200, twenty, GFP_KERNEL)?; + /// + /// let mut ma_lock = tree.lock(); + /// let mut iter = ma_lock.ma_state(0, usize::MAX); + /// + /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(10)); + /// assert_eq!(iter.find(usize::MAX).map(|v| *v), Some(20)); + /// assert!(iter.find(usize::MAX).is_none()); + /// # Ok::<_, Error>(()) + /// ``` + #[inline] + pub fn find(&mut self, max: usize) -> Option> { + let ret = self.mas_find_raw(max); + if ret.is_null() { + return None; + } + + // SAFETY: If the pointer is not null, then it references a valid instance of `T`. It's + // safe to access it mutably as the returned reference borrows this `MaState`, and the + // `MaState` has read/write access to the maple tree. + Some(unsafe { T::borrow_mut(ret) }) + } +} + +/// Error type for failure to insert a new value. +pub struct InsertError { + /// The value that could not be inserted. + pub value: T, + /// The reason for the failure to insert. + pub cause: InsertErrorKind, +} + +/// The reason for the failure to insert. +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum InsertErrorKind { + /// There is already a value in the requested range. + Occupied, + /// Failure to allocate memory. + AllocError(kernel::alloc::AllocError), + /// The insertion request was invalid. + InvalidRequest, +} + +impl From for Error { + #[inline] + fn from(kind: InsertErrorKind) -> Error { + match kind { + InsertErrorKind::Occupied => EEXIST, + InsertErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM, + InsertErrorKind::InvalidRequest => EINVAL, + } + } +} + +impl From> for Error { + #[inline] + fn from(insert_err: InsertError) -> Error { + Error::from(insert_err.cause) + } +} + +/// Error type for failure to insert a new value. +pub struct AllocError { + /// The value that could not be inserted. + pub value: T, + /// The reason for the failure to insert. + pub cause: AllocErrorKind, +} + +/// The reason for the failure to insert. +#[derive(PartialEq, Eq, Copy, Clone)] +pub enum AllocErrorKind { + /// There is not enough space for the requested allocation. + Busy, + /// Failure to allocate memory. + AllocError(kernel::alloc::AllocError), + /// The insertion request was invalid. + InvalidRequest, +} + +impl From for Error { + #[inline] + fn from(kind: AllocErrorKind) -> Error { + match kind { + AllocErrorKind::Busy => EBUSY, + AllocErrorKind::AllocError(kernel::alloc::AllocError) => ENOMEM, + AllocErrorKind::InvalidRequest => EINVAL, + } + } +} + +impl From> for Error { + #[inline] + fn from(insert_err: AllocError) -> Error { + Error::from(insert_err.cause) + } +} diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs new file mode 100644 index 000000000..8d4b719bd --- /dev/null +++ b/rust/kernel/miscdevice.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Miscdevice support. +//! +//! C headers: [`include/linux/miscdevice.h`](srctree/include/linux/miscdevice.h). +//! +//! Reference: + +use crate::{ + bindings, + device::Device, + error::{ + to_result, + VTABLE_DEFAULT_ERROR, // + }, + fs::{ + File, + Kiocb, // + }, + iov::{ + IovIterDest, + IovIterSource, // + }, + mm::virt::VmaNew, + module::this_module, + prelude::*, + seq_file::SeqFile, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::marker::PhantomData; + +/// Options for creating a misc device. +#[derive(Copy, Clone)] +pub struct MiscDeviceOptions { + /// The name of the miscdevice. + pub name: &'static CStr, +} + +impl MiscDeviceOptions { + /// Create a raw `struct miscdev` ready for registration. + pub const fn into_raw(self) -> bindings::miscdevice { + let mut result: bindings::miscdevice = pin_init::zeroed(); + result.minor = bindings::MISC_DYNAMIC_MINOR as ffi::c_int; + result.name = crate::str::as_char_ptr_in_const_context(self.name); + result.fops = MiscdeviceVTable::::build(); + result + } +} + +/// A registration of a miscdevice. +/// +/// # Invariants +/// +/// - `inner` contains a `struct miscdevice` that is registered using +/// `misc_register()`. +/// - This registration remains valid for the entire lifetime of the +/// [`MiscDeviceRegistration`] instance. +/// - Deregistration occurs exactly once in [`Drop`] via `misc_deregister()`. +/// - `inner` wraps a valid, pinned `miscdevice` created using +/// [`MiscDeviceOptions::into_raw`]. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct MiscDeviceRegistration { + #[pin] + inner: Opaque, + _t: PhantomData, +} + +// SAFETY: It is allowed to call `misc_deregister` on a different thread from where you called +// `misc_register`. +unsafe impl Send for MiscDeviceRegistration {} +// SAFETY: All `&self` methods on this type are written to ensure that it is safe to call them in +// parallel. +unsafe impl Sync for MiscDeviceRegistration {} + +impl MiscDeviceRegistration { + /// Register a misc device. + pub fn register(opts: MiscDeviceOptions) -> impl PinInit { + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(move |slot: *mut bindings::miscdevice| { + // SAFETY: The initializer can write to the provided `slot`. + unsafe { slot.write(opts.into_raw::()) }; + + // SAFETY: We just wrote the misc device options to the slot. The miscdevice will + // get unregistered before `slot` is deallocated because the memory is pinned and + // the destructor of this type deallocates the memory. + // INVARIANT: If this returns `Ok(())`, then the `slot` will contain a registered + // misc device. + to_result(unsafe { bindings::misc_register(slot) }) + }), + _t: PhantomData, + }) + } + + /// Returns a raw pointer to the misc device. + pub fn as_raw(&self) -> *mut bindings::miscdevice { + self.inner.get() + } + + /// Access the `this_device` field. + pub fn device(&self) -> &Device { + // SAFETY: This can only be called after a successful register(), which always + // initialises `this_device` with a valid device. Furthermore, the signature of this + // function tells the borrow-checker that the `&Device` reference must not outlive the + // `&MiscDeviceRegistration` used to obtain it, so the last use of the reference must be + // before the underlying `struct miscdevice` is destroyed. + unsafe { Device::from_raw((*self.as_raw()).this_device) } + } +} + +#[pinned_drop] +impl PinnedDrop for MiscDeviceRegistration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: We know that the device is registered by the type invariants. + unsafe { bindings::misc_deregister(self.inner.get()) }; + } +} + +/// Trait implemented by the private data of an open misc device. +#[vtable] +pub trait MiscDevice: Sized { + /// What kind of pointer should `Self` be wrapped in. + type Ptr: ForeignOwnable + Send + Sync; + + /// Called when the misc device is opened. + /// + /// The returned pointer will be stored as the private data for the file. + fn open(_file: &File, _misc: &MiscDeviceRegistration) -> Result; + + /// Called when the misc device is released. + fn release(device: Self::Ptr, _file: &File) { + drop(device); + } + + /// Handle for mmap. + /// + /// This function is invoked when a user space process invokes the `mmap` system call on + /// `file`. The function is a callback that is part of the VMA initializer. The kernel will do + /// initial setup of the VMA before calling this function. The function can then interact with + /// the VMA initialization by calling methods of `vma`. If the function does not return an + /// error, the kernel will complete initialization of the VMA according to the properties of + /// `vma`. + fn mmap( + _device: ::Borrowed<'_>, + _file: &File, + _vma: &VmaNew, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Read from this miscdevice. + fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Write to this miscdevice. + fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Handler for ioctls. + /// + /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`]. + /// + /// [`kernel::ioctl`]: mod@crate::ioctl + fn ioctl( + _device: ::Borrowed<'_>, + _file: &File, + _cmd: u32, + _arg: usize, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Handler for ioctls. + /// + /// Used for 32-bit userspace on 64-bit platforms. + /// + /// This method is optional and only needs to be provided if the ioctl relies on structures + /// that have different layout on 32-bit and 64-bit userspace. If no implementation is + /// provided, then `compat_ptr_ioctl` will be used instead. + #[cfg(CONFIG_COMPAT)] + fn compat_ioctl( + _device: ::Borrowed<'_>, + _file: &File, + _cmd: u32, + _arg: usize, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Show info for this fd. + fn show_fdinfo( + _device: ::Borrowed<'_>, + _m: &SeqFile, + _file: &File, + ) { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// A vtable for the file operations of a Rust miscdevice. +struct MiscdeviceVTable(PhantomData); + +impl MiscdeviceVTable { + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is undergoing initialization. + /// The file must be associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn open(inode: *mut bindings::inode, raw_file: *mut bindings::file) -> c_int { + // SAFETY: The pointers are valid and for a file being opened. + let ret = unsafe { bindings::generic_file_open(inode, raw_file) }; + if ret != 0 { + return ret; + } + + // SAFETY: The open call of a file can access the private data. + let misc_ptr = unsafe { (*raw_file).private_data }; + + // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the + // associated `struct miscdevice` before calling into this method. Furthermore, + // `misc_open()` ensures that the miscdevice can't be unregistered and freed during this + // call to `fops_open`. + let misc = unsafe { &*misc_ptr.cast::>() }; + + // SAFETY: + // * This underlying file is valid for (much longer than) the duration of `T::open`. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(raw_file) }; + + let ptr = match T::open(file, misc) { + Ok(ptr) => ptr, + Err(err) => return err.to_errno(), + }; + + // This overwrites the private data with the value specified by the user, changing the type + // of this file's private data. All future accesses to the private data is performed by + // other fops_* methods in this file, which all correctly cast the private data to the new + // type. + // + // SAFETY: The open call of a file can access the private data. + unsafe { (*raw_file).private_data = ptr.into_foreign() }; + + 0 + } + + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is being released. The file + /// must be associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn release(_inode: *mut bindings::inode, file: *mut bindings::file) -> c_int { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: The release call of a file owns the private data. + let ptr = unsafe { ::from_foreign(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + T::release(ptr, unsafe { File::from_raw_file(file) }); + + 0 + } + + /// # Safety + /// + /// `kiocb` must be correspond to a valid file that is associated with a + /// `MiscDeviceRegistration`. `iter` must be a valid `struct iov_iter` for writing. + unsafe extern "C" fn read_iter( + kiocb: *mut bindings::kiocb, + iter: *mut bindings::iov_iter, + ) -> isize { + // SAFETY: The caller provides a valid `struct kiocb` associated with a + // `MiscDeviceRegistration` file. + let kiocb = unsafe { Kiocb::from_raw(kiocb) }; + // SAFETY: This is a valid `struct iov_iter` for writing. + let iov = unsafe { IovIterDest::from_raw(iter) }; + + match T::read_iter(kiocb, iov) { + Ok(res) => res as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `kiocb` must be correspond to a valid file that is associated with a + /// `MiscDeviceRegistration`. `iter` must be a valid `struct iov_iter` for reading. + unsafe extern "C" fn write_iter( + kiocb: *mut bindings::kiocb, + iter: *mut bindings::iov_iter, + ) -> isize { + // SAFETY: The caller provides a valid `struct kiocb` associated with a + // `MiscDeviceRegistration` file. + let kiocb = unsafe { Kiocb::from_raw(kiocb) }; + // SAFETY: This is a valid `struct iov_iter` for reading. + let iov = unsafe { IovIterSource::from_raw(iter) }; + + match T::write_iter(kiocb, iov) { + Ok(res) => res as isize, + Err(err) => err.to_errno() as isize, + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + /// `vma` must be a vma that is currently being mmap'ed with this file. + unsafe extern "C" fn mmap( + file: *mut bindings::file, + vma: *mut bindings::vm_area_struct, + ) -> c_int { + // SAFETY: The mmap call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: This is a Rust Miscdevice, so we call `into_foreign` in `open` and + // `from_foreign` in `release`, and `fops_mmap` is guaranteed to be called between those + // two operations. + let device = unsafe { ::borrow(private.cast()) }; + // SAFETY: The caller provides a vma that is undergoing initial VMA setup. + let area = unsafe { VmaNew::from_raw(vma) }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::mmap(device, file, area) { + Ok(()) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + unsafe extern "C" fn ioctl(file: *mut bindings::file, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: The ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } + } + + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + #[cfg(CONFIG_COMPAT)] + unsafe extern "C" fn compat_ioctl( + file: *mut bindings::file, + cmd: c_uint, + arg: c_ulong, + ) -> c_long { + // SAFETY: The compat ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::compat_ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } + } + + /// # Safety + /// + /// - `file` must be a valid file that is associated with a `MiscDeviceRegistration`. + /// - `seq_file` must be a valid `struct seq_file` that we can write to. + unsafe extern "C" fn show_fdinfo(seq_file: *mut bindings::seq_file, file: *mut bindings::file) { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { ::borrow(private) }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in + // which this method is called. + let m = unsafe { SeqFile::from_raw(seq_file) }; + + T::show_fdinfo(device, m, file); + } + + const VTABLE: bindings::file_operations = bindings::file_operations { + open: Some(Self::open), + release: Some(Self::release), + mmap: if T::HAS_MMAP { Some(Self::mmap) } else { None }, + read_iter: if T::HAS_READ_ITER { + Some(Self::read_iter) + } else { + None + }, + write_iter: if T::HAS_WRITE_ITER { + Some(Self::write_iter) + } else { + None + }, + unlocked_ioctl: if T::HAS_IOCTL { + Some(Self::ioctl) + } else { + None + }, + #[cfg(CONFIG_COMPAT)] + compat_ioctl: if T::HAS_COMPAT_IOCTL { + Some(Self::compat_ioctl) + } else if T::HAS_IOCTL { + bindings::compat_ptr_ioctl + } else { + None + }, + show_fdinfo: if T::HAS_SHOW_FDINFO { + Some(Self::show_fdinfo) + } else { + None + }, + owner: this_module::().as_ptr(), + ..pin_init::zeroed() + }; + + const fn build() -> &'static bindings::file_operations { + &Self::VTABLE + } +} diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs new file mode 100644 index 000000000..4764d7b68 --- /dev/null +++ b/rust/kernel/mm.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Memory management. +//! +//! This module deals with managing the address space of userspace processes. Each process has an +//! instance of [`Mm`], which keeps track of multiple VMAs (virtual memory areas). Each VMA +//! corresponds to a region of memory that the userspace process can access, and the VMA lets you +//! control what happens when userspace reads or writes to that region of memory. +//! +//! C header: [`include/linux/mm.h`](srctree/include/linux/mm.h) + +use crate::{ + bindings, + sync::aref::{ARef, AlwaysRefCounted}, + types::{NotThreadSafe, Opaque}, +}; +use core::{ops::Deref, ptr::NonNull}; + +pub mod virt; +use virt::VmaRef; + +#[cfg(CONFIG_MMU)] +pub use mmput_async::MmWithUserAsync; +mod mmput_async; + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This represents the address space of a userspace process, so each process has one `Mm` +/// instance. It may hold many VMAs internally. +/// +/// There is a counter called `mm_users` that counts the users of the address space; this includes +/// the userspace process itself, but can also include kernel threads accessing the address space. +/// Once `mm_users` reaches zero, this indicates that the address space can be destroyed. To access +/// the address space, you must prevent `mm_users` from reaching zero while you are accessing it. +/// The [`MmWithUser`] type represents an address space where this is guaranteed, and you can +/// create one using [`mmget_not_zero`]. +/// +/// The `ARef` smart pointer holds an `mmgrab` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmgrab`. +/// +/// [`mmget_not_zero`]: Mm::mmget_not_zero +#[repr(transparent)] +pub struct Mm { + mm: Opaque, +} + +// SAFETY: It is safe to call `mmdrop` on another thread than where `mmgrab` was called. +unsafe impl Send for Mm {} +// SAFETY: All methods on `Mm` can be called in parallel from several threads. +unsafe impl Sync for Mm {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for Mm { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmgrab(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmdrop(obj.cast().as_ptr()) }; + } +} + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This type is like [`Mm`], but with non-zero `mm_users`. It can only be used when `mm_users` can +/// be proven to be non-zero at compile-time, usually because the relevant code holds an `mmget` +/// refcount. It can be used to access the associated address space. +/// +/// The `ARef` smart pointer holds an `mmget` refcount. Its destructor may sleep. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero. +#[repr(transparent)] +pub struct MmWithUser { + mm: Mm, +} + +// SAFETY: It is safe to call `mmput` on another thread than where `mmget` was called. +unsafe impl Send for MmWithUser {} +// SAFETY: All methods on `MmWithUser` can be called in parallel from several threads. +unsafe impl Sync for MmWithUser {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for MmWithUser { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmget(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmput(obj.cast().as_ptr()) }; + } +} + +// Make all `Mm` methods available on `MmWithUser`. +impl Deref for MmWithUser { + type Target = Mm; + + #[inline] + fn deref(&self) -> &Mm { + &self.mm + } +} + +// These methods are safe to call even if `mm_users` is zero. +impl Mm { + /// Returns a raw pointer to the inner `mm_struct`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::mm_struct { + self.mm.get() + } + + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that it is not deallocated + /// during the lifetime 'a. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a Mm { + // SAFETY: Caller promises that the pointer is valid for 'a. Layouts are compatible due to + // repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Calls `mmget_not_zero` and returns a handle if it succeeds. + #[inline] + pub fn mmget_not_zero(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmget_not_zero(self.as_raw()) }; + + if success { + // SAFETY: We just created an `mmget` refcount. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(self.as_raw().cast())) }) + } else { + None + } + } +} + +// These methods require `mm_users` to be non-zero. +impl MmWithUser { + /// Obtain a reference from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` points at an `mm_struct`, and that `mm_users` remains + /// non-zero for the duration of the lifetime 'a. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser { + // SAFETY: Caller promises that the pointer is valid for 'a. The layout is compatible due + // to repr(transparent). + unsafe { &*ptr.cast() } + } + + /// Attempt to access a vma using the vma read lock. + /// + /// This is an optimistic trylock operation, so it may fail if there is contention. In that + /// case, you should fall back to taking the mmap read lock. + /// + /// When per-vma locks are disabled, this always returns `None`. + #[inline] + pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { + #[cfg(CONFIG_PER_VMA_LOCK)] + { + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if !vma.is_null() { + return Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }); + } + } + + // Silence warnings about unused variables. + #[cfg(not(CONFIG_PER_VMA_LOCK))] + let _ = vma_addr; + + None + } + + /// Lock the mmap read lock. + #[inline] + pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmap_read_lock(self.as_raw()) }; + + // INVARIANT: We just acquired the read lock. + MmapReadGuard { + mm: self, + _nts: NotThreadSafe, + } + } + + /// Try to lock the mmap read lock. + #[inline] + pub fn mmap_read_trylock(&self) -> Option> { + // SAFETY: The pointer is valid since self is a reference. + let success = unsafe { bindings::mmap_read_trylock(self.as_raw()) }; + + if success { + // INVARIANT: We just acquired the read lock. + Some(MmapReadGuard { + mm: self, + _nts: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard for the mmap read lock. +/// +/// # Invariants +/// +/// This `MmapReadGuard` guard owns the mmap read lock. +pub struct MmapReadGuard<'a> { + mm: &'a MmWithUser, + // `mmap_read_lock` and `mmap_read_unlock` must be called on the same thread + _nts: NotThreadSafe, +} + +impl<'a> MmapReadGuard<'a> { + /// Look up a vma at the given address. + #[inline] + pub fn vma_lookup(&self, vma_addr: usize) -> Option<&virt::VmaRef> { + // SAFETY: By the type invariants we hold the mmap read guard, so we can safely call this + // method. Any value is okay for `vma_addr`. + let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr) }; + + if vma.is_null() { + None + } else { + // SAFETY: We just checked that a vma was found, so the pointer references a valid vma. + // + // Furthermore, the returned vma is still under the protection of the read lock guard + // and can be used while the mmap read lock is still held. That the vma is not used + // after the MmapReadGuard gets dropped is enforced by the borrow-checker. + unsafe { Some(virt::VmaRef::from_raw(vma)) } + } + } +} + +impl Drop for MmapReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) }; + } +} + +/// A guard for the vma read lock. +/// +/// # Invariants +/// +/// This `VmaReadGuard` guard owns the vma read lock. +pub struct VmaReadGuard<'a> { + vma: &'a VmaRef, + // `vma_end_read` must be called on the same thread as where the lock was taken + _nts: NotThreadSafe, +} + +// Make all `VmaRef` methods available on `VmaReadGuard`. +impl Deref for VmaReadGuard<'_> { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + self.vma + } +} + +impl Drop for VmaReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::vma_end_read(self.vma.as_ptr()) }; + } +} diff --git a/rust/kernel/mm/mmput_async.rs b/rust/kernel/mm/mmput_async.rs new file mode 100644 index 000000000..b8d2f0512 --- /dev/null +++ b/rust/kernel/mm/mmput_async.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Version of `MmWithUser` using `mmput_async`. +//! +//! This is a separate file from `mm.rs` due to the dependency on `CONFIG_MMU=y`. +#![cfg(CONFIG_MMU)] + +use crate::{ + bindings, + mm::MmWithUser, + sync::aref::{ARef, AlwaysRefCounted}, +}; +use core::{ops::Deref, ptr::NonNull}; + +/// A wrapper for the kernel's `struct mm_struct`. +/// +/// This type is identical to `MmWithUser` except that it uses `mmput_async` when dropping a +/// refcount. This means that the destructor of `ARef` is safe to call in atomic +/// context. +/// +/// # Invariants +/// +/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero. +#[repr(transparent)] +pub struct MmWithUserAsync { + mm: MmWithUser, +} + +// SAFETY: It is safe to call `mmput_async` on another thread than where `mmget` was called. +unsafe impl Send for MmWithUserAsync {} +// SAFETY: All methods on `MmWithUserAsync` can be called in parallel from several threads. +unsafe impl Sync for MmWithUserAsync {} + +// SAFETY: By the type invariants, this type is always refcounted. +unsafe impl AlwaysRefCounted for MmWithUserAsync { + #[inline] + fn inc_ref(&self) { + // SAFETY: The pointer is valid since self is a reference. + unsafe { bindings::mmget(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The caller is giving up their refcount. + unsafe { bindings::mmput_async(obj.cast().as_ptr()) }; + } +} + +// Make all `MmWithUser` methods available on `MmWithUserAsync`. +impl Deref for MmWithUserAsync { + type Target = MmWithUser; + + #[inline] + fn deref(&self) -> &MmWithUser { + &self.mm + } +} + +impl MmWithUser { + /// Use `mmput_async` when dropping this refcount. + #[inline] + pub fn into_mmput_async(me: ARef) -> ARef { + // SAFETY: The layouts and invariants are compatible. + unsafe { ARef::from_raw(ARef::into_raw(me).cast()) } + } +} diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs new file mode 100644 index 000000000..63eb730b0 --- /dev/null +++ b/rust/kernel/mm/virt.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Virtual memory. +//! +//! This module deals with managing a single VMA in the address space of a userspace process. Each +//! VMA corresponds to a region of memory that the userspace process can access, and the VMA lets +//! you control what happens when userspace reads or writes to that region of memory. +//! +//! The module has several different Rust types that all correspond to the C type called +//! `vm_area_struct`. The different structs represent what kind of access you have to the VMA, e.g. +//! [`VmaRef`] is used when you hold the mmap or vma read lock. Using the appropriate struct +//! ensures that you can't, for example, accidentally call a function that requires holding the +//! write lock when you only hold the read lock. + +use crate::{ + bindings, + error::{code::EINVAL, to_result, Result}, + mm::MmWithUser, + page::Page, + types::Opaque, +}; + +use core::ops::Deref; + +/// A wrapper for the kernel's `struct vm_area_struct` with read access. +/// +/// It represents an area of virtual memory. +/// +/// # Invariants +/// +/// The caller must hold the mmap read lock or the vma read lock. +#[repr(transparent)] +pub struct VmaRef { + vma: Opaque, +} + +// Methods you can call when holding the mmap or vma read lock (or stronger). They must be usable +// no matter what the vma flags are. +impl VmaRef { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap or vma + /// read lock (or stronger) is held for at least the duration of 'a. + #[inline] + pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Returns a raw pointer to this area. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::vm_area_struct { + self.vma.get() + } + + /// Access the underlying `mm_struct`. + #[inline] + pub fn mm(&self) -> &MmWithUser { + // SAFETY: By the type invariants, this `vm_area_struct` is valid and we hold the mmap/vma + // read lock or stronger. This implies that the underlying mm has a non-zero value of + // `mm_users`. + unsafe { MmWithUser::from_raw((*self.as_ptr()).vm_mm) } + } + + /// Returns the flags associated with the virtual memory area. + /// + /// The possible flags are a combination of the constants in [`flags`]. + #[inline] + pub fn flags(&self) -> vm_flags_t { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags } + } + + /// Returns the (inclusive) start address of the virtual memory area. + #[inline] + pub fn start(&self) -> usize { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_start } + } + + /// Returns the (exclusive) end address of the virtual memory area. + #[inline] + pub fn end(&self) -> usize { + // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this + // access is not a data race. + unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_end } + } + + /// Zap pages in the given page range. + /// + /// This clears page table mappings for the range at the leaf level, leaving all other page + /// tables intact, and freeing any memory referenced by the VMA in this range. That is, + /// anonymous memory is completely freed, file-backed memory has its reference count on page + /// cache folio's dropped, any dirty data will still be written back to disk as usual. + /// + /// It may seem odd that we clear at the leaf level, this is however a product of the page + /// table structure used to map physical memory into a virtual address space - each virtual + /// address actually consists of a bitmap of array indices into page tables, which form a + /// hierarchical page table level structure. + /// + /// As a result, each page table level maps a multiple of page table levels below, and thus + /// span ever increasing ranges of pages. At the leaf or PTE level, we map the actual physical + /// memory. + /// + /// It is here where a zap operates, as it the only place we can be certain of clearing without + /// impacting any other virtual mappings. It is an implementation detail as to whether the + /// kernel goes further in freeing unused page tables, but for the purposes of this operation + /// we must only assume that the leaf level is cleared. + #[inline] + pub fn zap_vma_range(&self, address: usize, size: usize) { + let (end, did_overflow) = address.overflowing_add(size); + if did_overflow || address < self.start() || self.end() < end { + // TODO: call WARN_ONCE once Rust version of it is added + return; + } + + // SAFETY: By the type invariants, the caller has read access to this VMA, which is + // sufficient for this method call. This method has no requirements on the vma flags. The + // address range is checked to be within the vma. + unsafe { bindings::zap_vma_range(self.as_ptr(), address, size) }; + } + + /// If the [`VM_MIXEDMAP`] flag is set, returns a [`VmaMixedMap`] to this VMA, otherwise + /// returns `None`. + /// + /// This can be used to access methods that require [`VM_MIXEDMAP`] to be set. + /// + /// [`VM_MIXEDMAP`]: flags::MIXEDMAP + #[inline] + pub fn as_mixedmap_vma(&self) -> Option<&VmaMixedMap> { + if self.flags() & flags::MIXEDMAP != 0 { + // SAFETY: We just checked that `VM_MIXEDMAP` is set. All other requirements are + // satisfied by the type invariants of `VmaRef`. + Some(unsafe { VmaMixedMap::from_raw(self.as_ptr()) }) + } else { + None + } + } +} + +/// A wrapper for the kernel's `struct vm_area_struct` with read access and [`VM_MIXEDMAP`] set. +/// +/// It represents an area of virtual memory. +/// +/// This struct is identical to [`VmaRef`] except that it must only be used when the +/// [`VM_MIXEDMAP`] flag is set on the vma. +/// +/// # Invariants +/// +/// The caller must hold the mmap read lock or the vma read lock. The `VM_MIXEDMAP` flag must be +/// set. +/// +/// [`VM_MIXEDMAP`]: flags::MIXEDMAP +#[repr(transparent)] +pub struct VmaMixedMap { + vma: VmaRef, +} + +// Make all `VmaRef` methods available on `VmaMixedMap`. +impl Deref for VmaMixedMap { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + &self.vma + } +} + +impl VmaMixedMap { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap read lock + /// (or stronger) is held for at least the duration of 'a. The `VM_MIXEDMAP` flag must be set. + #[inline] + pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Maps a single page at the given address within the virtual memory area. + /// + /// This operation does not take ownership of the page. + #[inline] + pub fn vm_insert_page(&self, address: usize, page: &Page) -> Result { + // SAFETY: By the type invariant of `Self` caller has read access and has verified that + // `VM_MIXEDMAP` is set. By invariant on `Page` the page has order 0. + to_result(unsafe { bindings::vm_insert_page(self.as_ptr(), address, page.as_ptr()) }) + } +} + +/// A configuration object for setting up a VMA in an `f_ops->mmap()` hook. +/// +/// The `f_ops->mmap()` hook is called when a new VMA is being created, and the hook is able to +/// configure the VMA in various ways to fit the driver that owns it. Using `VmaNew` indicates that +/// you are allowed to perform operations on the VMA that can only be performed before the VMA is +/// fully initialized. +/// +/// # Invariants +/// +/// For the duration of 'a, the referenced vma must be undergoing initialization in an +/// `f_ops->mmap()` hook. +#[repr(transparent)] +pub struct VmaNew { + vma: VmaRef, +} + +// Make all `VmaRef` methods available on `VmaNew`. +impl Deref for VmaNew { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + &self.vma + } +} + +impl VmaNew { + /// Access a virtual memory area given a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `vma` is undergoing initial vma setup for the duration of 'a. + #[inline] + pub unsafe fn from_raw<'a>(vma: *mut bindings::vm_area_struct) -> &'a Self { + // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a. + unsafe { &*vma.cast() } + } + + /// Internal method for updating the vma flags. + /// + /// # Safety + /// + /// This must not be used to set the flags to an invalid value. + #[inline] + unsafe fn update_flags(&self, set: vm_flags_t, unset: vm_flags_t) { + let mut flags = self.flags(); + flags |= set; + flags &= !unset; + + // SAFETY: This is not a data race: the vma is undergoing initial setup, so it's not yet + // shared. Additionally, `VmaNew` is `!Sync`, so it cannot be used to write in parallel. + // The caller promises that this does not set the flags to an invalid value. + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags = flags }; + } + + /// Set the `VM_MIXEDMAP` flag on this vma. + /// + /// This enables the vma to contain both `struct page` and pure PFN pages. Returns a reference + /// that can be used to call `vm_insert_page` on the vma. + #[inline] + pub fn set_mixedmap(&self) -> &VmaMixedMap { + // SAFETY: We don't yet provide a way to set VM_PFNMAP, so this cannot put the flags in an + // invalid state. + unsafe { self.update_flags(flags::MIXEDMAP, 0) }; + + // SAFETY: We just set `VM_MIXEDMAP` on the vma. + unsafe { VmaMixedMap::from_raw(self.vma.as_ptr()) } + } + + /// Set the `VM_IO` flag on this vma. + /// + /// This is used for memory mapped IO and similar. The flag tells other parts of the kernel to + /// avoid looking at the pages. For memory mapped IO this is useful as accesses to the pages + /// could have side effects. + #[inline] + pub fn set_io(&self) { + // SAFETY: Setting the VM_IO flag is always okay. + unsafe { self.update_flags(flags::IO, 0) }; + } + + /// Set the `VM_DONTEXPAND` flag on this vma. + /// + /// This prevents the vma from being expanded with `mremap()`. + #[inline] + pub fn set_dontexpand(&self) { + // SAFETY: Setting the VM_DONTEXPAND flag is always okay. + unsafe { self.update_flags(flags::DONTEXPAND, 0) }; + } + + /// Set the `VM_DONTCOPY` flag on this vma. + /// + /// This prevents the vma from being copied on fork. This option is only permanent if `VM_IO` + /// is set. + #[inline] + pub fn set_dontcopy(&self) { + // SAFETY: Setting the VM_DONTCOPY flag is always okay. + unsafe { self.update_flags(flags::DONTCOPY, 0) }; + } + + /// Set the `VM_DONTDUMP` flag on this vma. + /// + /// This prevents the vma from being included in core dumps. This option is only permanent if + /// `VM_IO` is set. + #[inline] + pub fn set_dontdump(&self) { + // SAFETY: Setting the VM_DONTDUMP flag is always okay. + unsafe { self.update_flags(flags::DONTDUMP, 0) }; + } + + /// Returns whether `VM_READ` is set. + /// + /// This flag indicates whether userspace is mapping this vma as readable. + #[inline] + pub fn readable(&self) -> bool { + (self.flags() & flags::READ) != 0 + } + + /// Try to clear the `VM_MAYREAD` flag, failing if `VM_READ` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma readable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYREAD` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_mayread(&self) -> Result { + if self.readable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYREAD` is okay when `VM_READ` is not set. + unsafe { self.update_flags(0, flags::MAYREAD) }; + Ok(()) + } + + /// Returns whether `VM_WRITE` is set. + /// + /// This flag indicates whether userspace is mapping this vma as writable. + #[inline] + pub fn writable(&self) -> bool { + (self.flags() & flags::WRITE) != 0 + } + + /// Try to clear the `VM_MAYWRITE` flag, failing if `VM_WRITE` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma writable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYWRITE` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_maywrite(&self) -> Result { + if self.writable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYWRITE` is okay when `VM_WRITE` is not set. + unsafe { self.update_flags(0, flags::MAYWRITE) }; + Ok(()) + } + + /// Returns whether `VM_EXEC` is set. + /// + /// This flag indicates whether userspace is mapping this vma as executable. + #[inline] + pub fn executable(&self) -> bool { + (self.flags() & flags::EXEC) != 0 + } + + /// Try to clear the `VM_MAYEXEC` flag, failing if `VM_EXEC` is set. + /// + /// This flag indicates whether userspace is allowed to make this vma executable with + /// `mprotect()`. + /// + /// Note that this operation is irreversible. Once `VM_MAYEXEC` has been cleared, it can never + /// be set again. + #[inline] + pub fn try_clear_mayexec(&self) -> Result { + if self.executable() { + return Err(EINVAL); + } + // SAFETY: Clearing `VM_MAYEXEC` is okay when `VM_EXEC` is not set. + unsafe { self.update_flags(0, flags::MAYEXEC) }; + Ok(()) + } +} + +/// The integer type used for vma flags. +#[doc(inline)] +pub use bindings::vm_flags_t; + +/// All possible flags for [`VmaRef`]. +pub mod flags { + use super::vm_flags_t; + use crate::bindings; + + /// No flags are set. + pub const NONE: vm_flags_t = bindings::VM_NONE as vm_flags_t; + + /// Mapping allows reads. + pub const READ: vm_flags_t = bindings::VM_READ as vm_flags_t; + + /// Mapping allows writes. + pub const WRITE: vm_flags_t = bindings::VM_WRITE as vm_flags_t; + + /// Mapping allows execution. + pub const EXEC: vm_flags_t = bindings::VM_EXEC as vm_flags_t; + + /// Mapping is shared. + pub const SHARED: vm_flags_t = bindings::VM_SHARED as vm_flags_t; + + /// Mapping may be updated to allow reads. + pub const MAYREAD: vm_flags_t = bindings::VM_MAYREAD as vm_flags_t; + + /// Mapping may be updated to allow writes. + pub const MAYWRITE: vm_flags_t = bindings::VM_MAYWRITE as vm_flags_t; + + /// Mapping may be updated to allow execution. + pub const MAYEXEC: vm_flags_t = bindings::VM_MAYEXEC as vm_flags_t; + + /// Mapping may be updated to be shared. + pub const MAYSHARE: vm_flags_t = bindings::VM_MAYSHARE as vm_flags_t; + + /// Page-ranges managed without `struct page`, just pure PFN. + pub const PFNMAP: vm_flags_t = bindings::VM_PFNMAP as vm_flags_t; + + /// Memory mapped I/O or similar. + pub const IO: vm_flags_t = bindings::VM_IO as vm_flags_t; + + /// Do not copy this vma on fork. + pub const DONTCOPY: vm_flags_t = bindings::VM_DONTCOPY as vm_flags_t; + + /// Cannot expand with mremap(). + pub const DONTEXPAND: vm_flags_t = bindings::VM_DONTEXPAND as vm_flags_t; + + /// Lock the pages covered when they are faulted in. + pub const LOCKONFAULT: vm_flags_t = bindings::VM_LOCKONFAULT as vm_flags_t; + + /// Is a VM accounted object. + pub const ACCOUNT: vm_flags_t = bindings::VM_ACCOUNT as vm_flags_t; + + /// Should the VM suppress accounting. + pub const NORESERVE: vm_flags_t = bindings::VM_NORESERVE as vm_flags_t; + + /// Huge TLB Page VM. + pub const HUGETLB: vm_flags_t = bindings::VM_HUGETLB as vm_flags_t; + + /// Synchronous page faults. (DAX-specific) + pub const SYNC: vm_flags_t = bindings::VM_SYNC as vm_flags_t; + + /// Architecture-specific flag. + pub const ARCH_1: vm_flags_t = bindings::VM_ARCH_1 as vm_flags_t; + + /// Wipe VMA contents in child on fork. + pub const WIPEONFORK: vm_flags_t = bindings::VM_WIPEONFORK as vm_flags_t; + + /// Do not include in the core dump. + pub const DONTDUMP: vm_flags_t = bindings::VM_DONTDUMP as vm_flags_t; + + /// Not soft dirty clean area. + pub const SOFTDIRTY: vm_flags_t = bindings::VM_SOFTDIRTY as vm_flags_t; + + /// Can contain `struct page` and pure PFN pages. + pub const MIXEDMAP: vm_flags_t = bindings::VM_MIXEDMAP as vm_flags_t; + + /// MADV_HUGEPAGE marked this vma. + pub const HUGEPAGE: vm_flags_t = bindings::VM_HUGEPAGE as vm_flags_t; + + /// MADV_NOHUGEPAGE marked this vma. + pub const NOHUGEPAGE: vm_flags_t = bindings::VM_NOHUGEPAGE as vm_flags_t; + + /// KSM may merge identical pages. + pub const MERGEABLE: vm_flags_t = bindings::VM_MERGEABLE as vm_flags_t; +} diff --git a/rust/kernel/module.rs b/rust/kernel/module.rs new file mode 100644 index 000000000..d71370598 --- /dev/null +++ b/rust/kernel/module.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Module-related types and helpers. + +/// The entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync + Send { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> crate::error::Result; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl pin_init::PinInit; +} + +impl InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit { + let initer = move |slot: *mut Self| { + let m = ::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { pin_init::pin_init_from_closure(initer) } + } +} + +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; + + /// The module's `THIS_MODULE` pointer. + const THIS_MODULE: ThisModule; +} + +/// Returns a reference to the `THIS_MODULE` of the given module type. +#[inline] +pub const fn this_module() -> &'static ThisModule { + &M::THIS_MODULE +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) +pub struct ThisModule(*mut crate::bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut crate::bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut crate::bindings::module { + self.0 + } +} diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 000000000..f9a14765a --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) + +use crate::prelude::*; +use crate::str::{kstrtobool_bytes, BStr}; +use bindings; +use kernel::sync::SetOnce; + +/// Newtype to make `bindings::kernel_param` [`Sync`]. +#[repr(transparent)] +#[doc(hidden)] +pub struct KernelParam(bindings::kernel_param); + +impl KernelParam { + #[doc(hidden)] + pub const fn new(val: bindings::kernel_param) -> Self { + Self(val) + } +} + +// SAFETY: C kernel handles serializing access to this type. We never access it +// from Rust module. +unsafe impl Sync for KernelParam {} + +/// Types that can be used for module parameters. +// NOTE: This trait is `Copy` because drop could produce unsoundness during teardown. +pub trait ModuleParam: Sized + Copy { + /// Parse a parameter argument into the parameter value. + fn try_from_param_arg(arg: &BStr) -> Result; +} + +/// Set the module parameter from a string. +/// +/// Used to set the parameter value at kernel initialization, when loading +/// the module or when set through `sysfs`. +/// +/// See `struct kernel_param_ops.set`. +/// +/// # Safety +/// +/// - If `val` is non-null then it must point to a valid null-terminated string that must be valid +/// for reads for the duration of the call. +/// - `param` must be a pointer to a `bindings::kernel_param` initialized by the rust module macro. +/// The pointee must be valid for reads for the duration of the call. +/// +/// # Note +/// +/// - The safety requirements are satisfied by C API contract when this function is invoked by the +/// module subsystem C code. +/// - Currently, we only support read-only parameters that are not readable from `sysfs`. Thus, this +/// function is only called at kernel initialization time, or at module load time, and we have +/// exclusive access to the parameter for the duration of the function. +/// +/// [`module!`]: macros::module +unsafe extern "C" fn set_param(val: *const c_char, param: *const bindings::kernel_param) -> c_int +where + T: ModuleParam, +{ + // NOTE: If we start supporting arguments without values, val _is_ allowed + // to be null here. + if val.is_null() { + crate::pr_warn_once!("Null pointer passed to `module_param::set_param`\n"); + return EINVAL.to_errno(); + } + + // SAFETY: By function safety requirement, val is non-null, null-terminated + // and valid for reads for the duration of this function. + let arg = unsafe { CStr::from_char_ptr(val) }; + let arg: &BStr = arg.as_ref(); + + crate::error::from_result(|| { + let new_value = T::try_from_param_arg(arg)?; + + // SAFETY: By function safety requirements, this access is safe. + let container = unsafe { &*((*param).__bindgen_anon_1.arg.cast::>()) }; + + container + .populate(new_value) + .then_some(0) + .ok_or(kernel::error::code::EEXIST) + }) +} + +macro_rules! impl_int_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + fn try_from_param_arg(arg: &BStr) -> Result { + <$ty as crate::str::parse_int::ParseInt>::from_str(arg) + } + } + }; +} + +impl_int_module_param!(i8); +impl_int_module_param!(u8); +impl_int_module_param!(i16); +impl_int_module_param!(u16); +impl_int_module_param!(i32); +impl_int_module_param!(u32); +impl_int_module_param!(i64); +impl_int_module_param!(u64); +impl_int_module_param!(isize); +impl_int_module_param!(usize); + +impl ModuleParam for bool { + fn try_from_param_arg(arg: &BStr) -> Result { + kstrtobool_bytes(arg) + } +} + +/// A wrapper for kernel parameters. +/// +/// This type is instantiated by the [`module!`] macro when module parameters are +/// defined. You should never need to instantiate this type directly. +/// +/// Note: This type is `pub` because it is used by module crates to access +/// parameter values. +pub struct ModuleParamAccess { + value: SetOnce, + default: T, +} + +// SAFETY: We only create shared references to the contents of this container, +// so if `T` is `Sync`, so is `ModuleParamAccess`. +unsafe impl Sync for ModuleParamAccess {} + +impl ModuleParamAccess { + #[doc(hidden)] + pub const fn new(default: T) -> Self { + Self { + value: SetOnce::new(), + default, + } + } + + /// Get a copy of the parameter value. + /// + /// Returns the value supplied at module load time, or the default value + /// if the parameter has not been set. + #[inline] + pub fn value(&self) -> T + where + T: Copy, + { + self.value.copy().unwrap_or(self.default) + } + + /// Get a shared reference to the parameter value. + /// + /// Returns a reference to the value supplied at module load time, or a + /// reference to the default value if the parameter has not been set. + // Note: When sysfs access to parameters are enabled, we have to pass in a + // held lock guard here. + #[inline] + pub fn value_ref(&self) -> &T { + self.value.as_ref().unwrap_or(&self.default) + } + + /// Get a mutable pointer to `self`. + /// + /// NOTE: In most cases it is not safe deref the returned pointer. + pub const fn as_void_ptr(&self) -> *mut c_void { + core::ptr::from_ref(self).cast_mut().cast() + } +} + +#[doc(hidden)] +/// Generate a static [`kernel_param_ops`](srctree/include/linux/moduleparam.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + #[doc(hidden)] + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: 0, + set: Some(set_param::<$ty>), + get: None, + free: None, + }; + }; +} + +make_param_ops!(PARAM_OPS_I8, i8); +make_param_ops!(PARAM_OPS_U8, u8); +make_param_ops!(PARAM_OPS_I16, i16); +make_param_ops!(PARAM_OPS_U16, u16); +make_param_ops!(PARAM_OPS_I32, i32); +make_param_ops!(PARAM_OPS_U32, u32); +make_param_ops!(PARAM_OPS_I64, i64); +make_param_ops!(PARAM_OPS_U64, u64); +make_param_ops!(PARAM_OPS_ISIZE, isize); +make_param_ops!(PARAM_OPS_USIZE, usize); +make_param_ops!(PARAM_OPS_BOOL, bool); diff --git a/rust/kernel/net/mod.rs b/rust/kernel/net/mod.rs new file mode 100644 index 000000000..8ecae7577 --- /dev/null +++ b/rust/kernel/net/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Networking. + +#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] +pub mod phy; + +pub mod netlink; diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs new file mode 100644 index 000000000..22ef3dde3 --- /dev/null +++ b/rust/kernel/net/netlink.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2026 Google LLC. + +//! Rust support for generic netlink. +//! +//! Currently only supports exposing multicast groups. +//! +//! C header: [`include/net/genetlink.h`](srctree/include/net/genetlink.h) + +use kernel::{ + alloc::{self, AllocError}, + error::to_result, + prelude::*, + transmute::AsBytes, + types::Opaque, + ThisModule, +}; + +use core::{ + mem::ManuallyDrop, + ptr::NonNull, // +}; + +/// The default netlink message size. +pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE; + +/// A wrapper around `struct sk_buff` for generic netlink messages. +/// +/// This type is intended to be specific for buffers used with netlink only, and other usecases for +/// `struct sk_buff` are out-of-scope for this abstraction. +/// +/// # Invariants +/// +/// The pointer has ownership over a valid `sk_buff`. +pub struct NetlinkSkBuff { + skb: NonNull, +} + +impl NetlinkSkBuff { + /// Creates a new `NetlinkSkBuff` with the given size. + pub fn new(size: usize, flags: alloc::Flags) -> Result { + // SAFETY: `genlmsg_new` only requires its arguments to be valid integers. + let skb = unsafe { bindings::genlmsg_new(size, flags.as_raw()) }; + let skb = NonNull::new(skb).ok_or(AllocError)?; + Ok(NetlinkSkBuff { skb }) + } + + /// Puts a generic netlink header into the `NetlinkSkBuff`. + pub fn genlmsg_put( + self, + portid: u32, + seq: u32, + family: &'static Family, + cmd: u8, + ) -> Result { + let skb = self.skb.as_ptr(); + // SAFETY: The skb and family pointers are valid. + let hdr = unsafe { bindings::genlmsg_put(skb, portid, seq, family.as_raw(), 0, cmd) }; + let hdr = NonNull::new(hdr).ok_or(AllocError)?; + Ok(GenlMsg { skb: self, hdr }) + } +} + +impl Drop for NetlinkSkBuff { + fn drop(&mut self) { + // SAFETY: We have ownership over the `sk_buff`, so we may free it. + unsafe { bindings::nlmsg_free(self.skb.as_ptr()) } + } +} + +/// A generic netlink message being constructed. +/// +/// # Invariants +/// +/// `hdr` references the header in this netlink message. +pub struct GenlMsg { + skb: NetlinkSkBuff, + hdr: NonNull, +} + +impl GenlMsg { + /// Puts an attribute into the message. + #[inline] + fn put(&mut self, attrtype: c_int, value: &T) -> Result + where + T: ?Sized + AsBytes, + { + let skb = self.skb.skb.as_ptr(); + let len = size_of_val(value); + let ptr = core::ptr::from_ref(value).cast::(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is + // readable and initialized for its `size_of` bytes. + to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) + } + + /// Puts a `u32` attribute into the message. + #[inline] + pub fn put_u32(&mut self, attrtype: c_int, value: u32) -> Result { + self.put(attrtype, &value) + } + + /// Puts a string attribute into the message. + #[inline] + pub fn put_string(&mut self, attrtype: c_int, value: &CStr) -> Result { + self.put(attrtype, value.to_bytes_with_nul()) + } + + /// Puts a flag attribute into the message. + #[inline] + pub fn put_flag(&mut self, attrtype: c_int) -> Result { + let skb = self.skb.skb.as_ptr(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and a null pointer is valid + // when the length is zero. + to_result(unsafe { bindings::nla_put(skb, attrtype, 0, core::ptr::null()) }) + } + + /// Sends the generic netlink message as a multicast message. + #[inline] + pub fn multicast( + self, + family: &'static Family, + portid: u32, + group: u32, + flags: alloc::Flags, + ) -> Result { + let me = ManuallyDrop::new(self); + // SAFETY: The `skb` and `family` pointers are valid. We pass ownership of the `skb` to + // `genlmsg_multicast` by not dropping `self`. + unsafe { + bindings::genlmsg_end(me.skb.skb.as_ptr(), me.hdr.as_ptr()); + to_result(bindings::genlmsg_multicast( + family.as_raw(), + me.skb.skb.as_ptr(), + portid, + group, + flags.as_raw(), + )) + } + } +} +impl Drop for GenlMsg { + fn drop(&mut self) { + // SAFETY: The `hdr` pointer references the header of this generic netlink message. + unsafe { bindings::genlmsg_cancel(self.skb.skb.as_ptr(), self.hdr.as_ptr()) }; + } +} + +/// Flags for a generic netlink family. +struct FamilyFlags { + /// Whether the family supports network namespaces. + netnsok: bool, + /// Whether the family supports parallel operations. + parallel_ops: bool, +} + +impl FamilyFlags { + /// Converts the flags to the bitfield representation used by `genl_family`. + const fn into_bitfield(self) -> bindings::__BindgenBitfieldUnit<[u8; 1]> { + // The below shifts are verified correct by test_family_flags_bitfield() below. + // + // Although bindgen generates helpers to change bitfields based on the C headers, these + // helpers unfortunately can't be used in const context. Since `Family` needs to be filled + // out at build-time, we use this helper instead. + let mut bits = 0; + if self.netnsok { + bits |= 1 << 0; + } + if self.parallel_ops { + bits |= 1 << 1; + } + // Convert from little endian to the target's endianness. + bits = u8::from_le(bits); + // SAFETY: This bitfield is represented as an u8. + unsafe { core::mem::transmute::>(bits) } + } +} + +/// A generic netlink family. +#[repr(transparent)] +pub struct Family { + inner: Opaque, +} + +// SAFETY: The `Family` type is thread safe. +unsafe impl Sync for Family {} + +impl Family { + /// Creates a new `Family` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + /// + /// The name must be a nul-terminated string, but it is taken as `&[u8]` so that it can be used + /// more conveniently with the strings generated by bindgen. + pub const fn const_new( + module: &ThisModule, + name: &[u8], + version: u32, + mcgrps: &'static [MulticastGroup], + ) -> Family { + let n_mcgrps = mcgrps.len() as u8; + if n_mcgrps as usize != mcgrps.len() { + panic!("too many mcgrps"); + } + let mut genl_family = bindings::genl_family { + version, + _bitfield_1: FamilyFlags { + netnsok: true, + parallel_ops: true, + } + .into_bitfield(), + module: module.as_ptr(), + mcgrps: mcgrps.as_ptr().cast(), + n_mcgrps, + ..pin_init::zeroed() + }; + if CStr::from_bytes_with_nul(name).is_err() { + panic!("genl_family name not nul-terminated"); + } + if genl_family.name.len() < name.len() { + panic!("genl_family name too long"); + } + let mut i = 0; + while i < name.len() { + genl_family.name[i] = name[i]; + i += 1; + } + Family { + inner: Opaque::new(genl_family), + } + } + + /// Checks if there are any listeners for the given multicast group. + pub fn has_listeners(&self, group: u32) -> bool { + // SAFETY: The family and init_net pointers are valid. + unsafe { + bindings::genl_has_listeners(self.as_raw(), &raw mut bindings::init_net, group) != 0 + } + } + + /// Returns a raw pointer to the underlying `genl_family` structure. + pub fn as_raw(&self) -> *mut bindings::genl_family { + self.inner.get() + } +} + +/// A generic netlink multicast group. +#[repr(transparent)] +pub struct MulticastGroup { + // No Opaque because fully immutable + group: bindings::genl_multicast_group, +} + +// SAFETY: Pure data so thread safe. +unsafe impl Sync for MulticastGroup {} + +impl MulticastGroup { + /// Creates a new `MulticastGroup` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + pub const fn const_new(name: &CStr) -> MulticastGroup { + let mut group: bindings::genl_multicast_group = pin_init::zeroed(); + + let name = name.to_bytes_with_nul(); + if group.name.len() < name.len() { + panic!("genl_multicast_group name too long"); + } + let mut i = 0; + while i < name.len() { + group.name[i] = name[i]; + i += 1; + } + + MulticastGroup { group } + } +} + +/// A registration of a generic netlink family. +/// +/// This type represents the registration of a [`Family`]. When an instance of this type is +/// dropped, its respective generic netlink family will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.family` always holds a valid reference to an initialized and registered [`Family`]. +pub struct Registration { + family: &'static Family, +} + +impl Family { + /// Registers the generic netlink family with the kernel. + pub fn register(&'static self) -> Result { + // SAFETY: `self.as_raw()` is a valid pointer to a `genl_family` struct. + // The `genl_family` struct is static, so it will outlive the registration. + to_result(unsafe { bindings::genl_register_family(self.as_raw()) })?; + Ok(Registration { family: self }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `self.family.as_raw()` is a valid pointer to a registered `genl_family` struct. + // The `Registration` struct ensures that `genl_unregister_family` is called exactly once + // for this family when it goes out of scope. + unsafe { bindings::genl_unregister_family(self.family.as_raw()) }; + } +} + +#[macros::kunit_tests(rust_netlink)] +mod tests { + use super::*; + + #[test] + fn test_family_flags_bitfield() { + for netnsok in [false, true] { + for parallel_ops in [false, true] { + let mut b_fam = bindings::genl_family { + ..Default::default() + }; + b_fam.set_netnsok(if netnsok { 1 } else { 0 }); + b_fam.set_parallel_ops(if parallel_ops { 1 } else { 0 }); + + let c_bitfield = FamilyFlags { + netnsok, + parallel_ops, + } + .into_bitfield(); + + // SAFETY: The bit field is stored as u8. + let b_val: u8 = unsafe { core::mem::transmute(b_fam._bitfield_1) }; + // SAFETY: The bit field is stored as u8. + let c_val: u8 = unsafe { core::mem::transmute(c_bitfield) }; + assert_eq!(b_val, c_val); + } + } + } +} diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs new file mode 100644 index 000000000..c4e7b1d6c --- /dev/null +++ b/rust/kernel/net/phy.rs @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2023 FUJITA Tomonori + +//! Network PHY device. +//! +//! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). + +use crate::{device_id::RawDeviceId, error::*, prelude::*, types::Opaque}; +use core::{marker::PhantomData, ptr::addr_of_mut}; + +pub mod reg; + +/// PHY state machine states. +/// +/// Corresponds to the kernel's [`enum phy_state`]. +/// +/// Some of PHY drivers access to the state of PHY's software state machine. +/// +/// [`enum phy_state`]: srctree/include/linux/phy.h +#[derive(PartialEq, Eq)] +pub enum DeviceState { + /// PHY device and driver are not ready for anything. + Down, + /// PHY is ready to send and receive packets. + Ready, + /// PHY is up, but no polling or interrupts are done. + Halted, + /// PHY is up, but is in an error state. + Error, + /// PHY and attached device are ready to do work. + Up, + /// PHY is currently running. + Running, + /// PHY is up, but not currently plugged in. + NoLink, + /// PHY is performing a cable test. + CableTest, +} + +/// A mode of Ethernet communication. +/// +/// PHY drivers get duplex information from hardware and update the current state. +pub enum DuplexMode { + /// PHY is in full-duplex mode. + Full, + /// PHY is in half-duplex mode. + Half, + /// PHY is in unknown duplex mode. + Unknown, +} + +/// An instance of a PHY device. +/// +/// Wraps the kernel's [`struct phy_device`]. +/// +/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver +/// executes [`Driver`]'s methods during the callback. +/// +/// # Invariants +/// +/// - Referencing a `phy_device` using this struct asserts that you are in +/// a context where all methods defined on this struct are safe to call. +/// - This struct always has a valid `self.0.mdio.dev`. +/// +/// [`struct phy_device`]: srctree/include/linux/phy.h +// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is +// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for +// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with +// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance. +// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access +// to the instance. +#[repr(transparent)] +pub struct Device(Opaque); + +impl Device { + /// Creates a new [`Device`] instance from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, + /// - the pointer must point at a valid `phy_device`, and the caller + /// must be in a context where all methods defined on this struct + /// are safe to call. + /// - `(*ptr).mdio.dev` must be a valid. + unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self { + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`. + let ptr = ptr.cast::(); + // SAFETY: by the function requirements the pointer is valid and we have unique access for + // the duration of `'a`. + unsafe { &mut *ptr } + } + + /// Gets the id of the PHY. + pub fn phy_id(&self) -> u32 { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).phy_id } + } + + /// Gets the state of PHY state machine states. + pub fn state(&self) -> DeviceState { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let state = unsafe { (*phydev).state }; + // TODO: this conversion code will be replaced with automatically generated code by bindgen + // when it becomes possible. + match state { + bindings::phy_state_PHY_DOWN => DeviceState::Down, + bindings::phy_state_PHY_READY => DeviceState::Ready, + bindings::phy_state_PHY_HALTED => DeviceState::Halted, + bindings::phy_state_PHY_ERROR => DeviceState::Error, + bindings::phy_state_PHY_UP => DeviceState::Up, + bindings::phy_state_PHY_RUNNING => DeviceState::Running, + bindings::phy_state_PHY_NOLINK => DeviceState::NoLink, + bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest, + _ => DeviceState::Error, + } + } + + /// Gets the current link state. + /// + /// It returns true if the link is up. + #[inline] + pub fn is_link_up(&self) -> bool { + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let link = unsafe { bindings::phy_device::link_raw(phydev) }; + link == 1 + } + + /// Gets the current auto-negotiation configuration. + /// + /// It returns true if auto-negotiation is enabled. + #[inline] + pub fn is_autoneg_enabled(&self) -> bool { + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let autoneg = unsafe { bindings::phy_device::autoneg_raw(phydev) }; + autoneg == bindings::AUTONEG_ENABLE + } + + /// Gets the current auto-negotiation state. + /// + /// It returns true if auto-negotiation is completed. + #[inline] + pub fn is_autoneg_completed(&self) -> bool { + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let completed = unsafe { bindings::phy_device::autoneg_complete_raw(phydev) }; + completed == 1 + } + + /// Sets the speed of the PHY. + pub fn set_speed(&mut self, speed: u32) { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).speed = speed as c_int }; + } + + /// Sets duplex mode. + pub fn set_duplex(&mut self, mode: DuplexMode) { + let phydev = self.0.get(); + let v = match mode { + DuplexMode::Full => bindings::DUPLEX_FULL, + DuplexMode::Half => bindings::DUPLEX_HALF, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN, + }; + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).duplex = v as c_int }; + } + + /// Reads a PHY register. + // This function reads a hardware register and updates the stats so takes `&mut self`. + pub fn read(&mut self, reg: R) -> Result { + reg.read(self) + } + + /// Writes a PHY register. + pub fn write(&mut self, reg: R, val: u16) -> Result { + reg.write(self, val) + } + + /// Reads a paged register. + pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) }; + + to_result(ret).map(|()| ret as u16) + } + + /// Resolves the advertisements into PHY settings. + pub fn resolve_aneg_linkmode(&mut self) { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + unsafe { bindings::phy_resolve_aneg_linkmode(phydev) }; + } + + /// Executes software reset the PHY via `BMCR_RESET` bit. + pub fn genphy_soft_reset(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_soft_reset(phydev) }) + } + + /// Initializes the PHY. + pub fn init_hw(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::phy_init_hw(phydev) }) + } + + /// Starts auto-negotiation. + pub fn start_aneg(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::_phy_start_aneg(phydev) }) + } + + /// Resumes the PHY via `BMCR_PDOWN` bit. + pub fn genphy_resume(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_resume(phydev) }) + } + + /// Suspends the PHY via `BMCR_PDOWN` bit. + pub fn genphy_suspend(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_suspend(phydev) }) + } + + /// Checks the link status and updates current link state. + pub fn genphy_read_status(&mut self) -> Result { + R::read_status(self) + } + + /// Updates the link status. + pub fn genphy_update_link(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_update_link(phydev) }) + } + + /// Reads link partner ability. + pub fn genphy_read_lpa(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_lpa(phydev) }) + } + + /// Reads PHY abilities. + pub fn genphy_read_abilities(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_abilities(phydev) }) + } +} + +impl AsRef for Device { + fn as_ref(&self) -> &kernel::device::Device { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that `mdio.dev` is valid. + unsafe { kernel::device::Device::from_raw(addr_of_mut!((*phydev).mdio.dev)) } + } +} + +/// Defines certain other features this PHY supports (like interrupts). +/// +/// These flag values are used in [`Driver::FLAGS`]. +pub mod flags { + /// PHY is internal. + pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL; + /// PHY needs to be reset after the refclk is enabled. + pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN; + /// Polling is used to detect PHY status changes. + pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST; + /// Don't suspend. + pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND; +} + +/// An adapter for the registration of a PHY driver. +struct Adapter { + _p: PhantomData, +} + +impl Adapter { + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn soft_reset_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::soft_reset(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we can exclusively access `phy_device` because + // it's not published yet, so the accessors on `Device` are okay + // to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::probe(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn get_features_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::get_features(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::suspend(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::resume(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn config_aneg_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::config_aneg(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_status_callback(phydev: *mut bindings::phy_device) -> c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::read_status(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn match_phy_device_callback( + phydev: *mut bindings::phy_device, + _phydrv: *const bindings::phy_driver, + ) -> c_int { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::match_phy_device(dev).into() + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + // CAST: the C side verifies devnum < 32. + let ret = T::read_mmd(dev, devnum as u8, regnum)?; + Ok(ret.into()) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn write_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + val: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::write_mmd(dev, devnum as u8, regnum, val)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::link_change_notify(dev); + } +} + +/// Driver structure for a particular PHY type. +/// +/// Wraps the kernel's [`struct phy_driver`]. +/// This is used to register a driver for a particular PHY type with the kernel. +/// +/// # Invariants +/// +/// `self.0` is always in a valid state. +/// +/// [`struct phy_driver`]: srctree/include/linux/phy.h +#[repr(transparent)] +pub struct DriverVTable(Opaque); + +// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to +// share `&DriverVTable` across execution context boundaries. +unsafe impl Sync for DriverVTable {} + +/// Creates a [`DriverVTable`] instance from [`Driver`]. +/// +/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`. +/// +/// [`module_phy_driver`]: crate::module_phy_driver +pub const fn create_phy_driver() -> DriverVTable { + // INVARIANT: All the fields of `struct phy_driver` are initialized properly. + DriverVTable(Opaque::new(bindings::phy_driver { + name: crate::str::as_char_ptr_in_const_context(T::NAME).cast_mut(), + flags: T::FLAGS, + phy_id: T::PHY_DEVICE_ID.id(), + phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), + soft_reset: if T::HAS_SOFT_RESET { + Some(Adapter::::soft_reset_callback) + } else { + None + }, + probe: if T::HAS_PROBE { + Some(Adapter::::probe_callback) + } else { + None + }, + get_features: if T::HAS_GET_FEATURES { + Some(Adapter::::get_features_callback) + } else { + None + }, + match_phy_device: if T::HAS_MATCH_PHY_DEVICE { + Some(Adapter::::match_phy_device_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Adapter::::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Adapter::::resume_callback) + } else { + None + }, + config_aneg: if T::HAS_CONFIG_ANEG { + Some(Adapter::::config_aneg_callback) + } else { + None + }, + read_status: if T::HAS_READ_STATUS { + Some(Adapter::::read_status_callback) + } else { + None + }, + read_mmd: if T::HAS_READ_MMD { + Some(Adapter::::read_mmd_callback) + } else { + None + }, + write_mmd: if T::HAS_WRITE_MMD { + Some(Adapter::::write_mmd_callback) + } else { + None + }, + link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY { + Some(Adapter::::link_change_notify_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct phy_driver`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::::zeroed().assume_init() } + })) +} + +/// Driver implementation for a particular PHY type. +/// +/// This trait is used to create a [`DriverVTable`]. +#[vtable] +pub trait Driver { + /// Defines certain other features this PHY supports. + /// It is a combination of the flags in the [`flags`] module. + const FLAGS: u32 = 0; + + /// The friendly name of this PHY type. + const NAME: &'static CStr; + + /// This driver only works for PHYs with IDs which match this field. + /// The default id and mask are zero. + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0); + + /// Issues a PHY software reset. + fn soft_reset(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Sets up device-specific structures during discovery. + fn probe(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Probes the hardware to determine what abilities it has. + fn get_features(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Returns true if this is a suitable driver for the given phydev. + /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`]. + fn match_phy_device(_dev: &Device) -> bool { + false + } + + /// Configures the advertisement and resets auto-negotiation + /// if auto-negotiation is enabled. + fn config_aneg(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Determines the negotiated speed and duplex. + fn read_status(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Suspends the hardware, saving state if needed. + fn suspend(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Resumes the hardware, restoring state if needed. + fn resume(_dev: &mut Device) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD read function for reading a MMD register. + fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD write function for writing a MMD register. + fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Callback for notification of link change. + fn link_change_notify(_dev: &mut Device) {} +} + +/// Registration structure for PHY drivers. +/// +/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped. +/// +/// # Invariants +/// +/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`. +pub struct Registration { + drivers: Pin<&'static mut [DriverVTable]>, +} + +// SAFETY: The only action allowed in a `Registration` instance is dropping it, which is safe to do +// from any thread because `phy_drivers_unregister` can be called from any thread context. +unsafe impl Send for Registration {} + +impl Registration { + /// Registers a PHY driver. + pub fn register( + module: &'static crate::ThisModule, + drivers: Pin<&'static mut [DriverVTable]>, + ) -> Result { + if drivers.is_empty() { + return Err(code::EINVAL); + } + // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of + // the `drivers` slice are initialized properly. `drivers` will not be moved. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_drivers_register( + drivers[0].0.get(), + drivers.len().try_into()?, + module.as_ptr(), + ) + })?; + // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. + Ok(Registration { drivers }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.drivers` is valid. + // So it's just an FFI call. + unsafe { + bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32) + }; + } +} + +/// An identifier for PHY devices on an MDIO/MII bus. +/// +/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate +/// PHY driver. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::mdio_device_id); + +impl DeviceId { + /// Creates a new instance with the exact match mask. + pub const fn new_with_exact_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Exact.as_int(), + }) + } + + /// Creates a new instance with the model match mask. + pub const fn new_with_model_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Model.as_int(), + }) + } + + /// Creates a new instance with the vendor match mask. + pub const fn new_with_vendor_mask(id: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Vendor.as_int(), + }) + } + + /// Creates a new instance with a custom match mask. + pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Custom(mask).as_int(), + }) + } + + /// Creates a new instance from [`Driver`]. + pub const fn new_with_driver() -> Self { + T::PHY_DEVICE_ID + } + + /// Get the MDIO device's PHY ID. + pub const fn id(&self) -> u32 { + self.0.phy_id + } + + /// Get the MDIO device's match mask. + pub const fn mask_as_int(&self) -> u32 { + self.0.phy_id_mask + } + + // macro use only + #[doc(hidden)] + pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { + self.0 + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct mdio_device_id` +// and does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::mdio_device_id; +} + +enum DeviceMask { + Exact, + Model, + Vendor, + Custom(u32), +} + +impl DeviceMask { + const MASK_EXACT: u32 = !0; + const MASK_MODEL: u32 = !0 << 4; + const MASK_VENDOR: u32 = !0 << 10; + + const fn as_int(&self) -> u32 { + match self { + DeviceMask::Exact => Self::MASK_EXACT, + DeviceMask::Model => Self::MASK_MODEL, + DeviceMask::Vendor => Self::MASK_VENDOR, + DeviceMask::Custom(mask) => *mask, + } + } +} + +/// Declares a kernel module for PHYs drivers. +/// +/// This creates a static array of kernel's `struct phy_driver` and registers it. +/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information +/// for module loading into the module binary file. Every driver needs an entry in `device_table`. +/// +/// # Examples +/// +/// ``` +/// # mod module_phy_driver_sample { +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// kernel::module_phy_driver! { +/// drivers: [PhySample], +/// device_table: [ +/// DeviceId::new_with_driver::() +/// ], +/// name: "rust_sample_phy", +/// authors: ["Rust for Linux Contributors"], +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c"PhySample"; +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// # } +/// ``` +#[macro_export] +macro_rules! module_phy_driver { + (@replace_expr $_t:tt $sub:expr) => {$sub}; + + (@count_devices $($x:expr),*) => { + 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))* + }; + + (@device_table [$($dev:expr),+]) => { + $crate::module_device_table!( + "mdio", $crate::net::phy::DeviceId, + TABLE, @none, [$($dev),+] + ); + }; + + (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { + struct Module { + _reg: $crate::net::phy::Registration, + } + + $crate::prelude::module! { + type: Module, + $($f)* + } + + const _: () = { + static mut DRIVERS: [$crate::net::phy::DriverVTable; + $crate::module_phy_driver!(@count_devices $($driver),+)] = + [$($crate::net::phy::create_phy_driver::<$driver>()),+]; + + impl $crate::Module for Module { + fn init(module: &'static $crate::ThisModule) -> Result { + // SAFETY: The anonymous constant guarantees that nobody else can access + // the `DRIVERS` static. The array is used only in the C side. + let drivers = unsafe { &mut DRIVERS }; + let mut reg = $crate::net::phy::Registration::register( + module, + ::core::pin::Pin::static_mut(drivers), + )?; + Ok(Module { _reg: reg }) + } + } + }; + + $crate::module_phy_driver!(@device_table [$($dev),+]); + } +} diff --git a/rust/kernel/net/phy/reg.rs b/rust/kernel/net/phy/reg.rs new file mode 100644 index 000000000..80e22c264 --- /dev/null +++ b/rust/kernel/net/phy/reg.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 FUJITA Tomonori + +//! PHY register interfaces. +//! +//! This module provides support for accessing PHY registers in the +//! Ethernet management interface clauses 22 and 45 register namespaces, as +//! defined in IEEE 802.3. + +use super::Device; +use crate::{ + build_assert::build_assert, + error::*, + uapi, // +}; + +mod private { + /// Marker that a trait cannot be implemented outside of this crate + pub trait Sealed {} +} + +/// Accesses PHY registers. +/// +/// This trait is used to implement the unified interface to access +/// C22 and C45 PHY registers. +/// +/// # Examples +/// +/// ```ignore +/// fn link_change_notify(dev: &mut Device) { +/// // read C22 BMCR register +/// dev.read(C22::BMCR); +/// // read C45 PMA/PMD control 1 register +/// dev.read(C45::new(Mmd::PMAPMD, 0)); +/// +/// // Checks the link status as reported by registers in the C22 namespace +/// // and updates current link state. +/// dev.genphy_read_status::(); +/// // Checks the link status as reported by registers in the C45 namespace +/// // and updates current link state. +/// dev.genphy_read_status::(); +/// } +/// ``` +pub trait Register: private::Sealed { + /// Reads a PHY register. + fn read(&self, dev: &mut Device) -> Result; + + /// Writes a PHY register. + fn write(&self, dev: &mut Device, val: u16) -> Result; + + /// Checks the link status and updates current link state. + fn read_status(dev: &mut Device) -> Result; +} + +/// A single MDIO clause 22 register address (5 bits). +#[derive(Copy, Clone, Debug)] +pub struct C22(u8); + +impl C22 { + /// Basic mode control. + pub const BMCR: Self = C22(0x00); + /// Basic mode status. + pub const BMSR: Self = C22(0x01); + /// PHY identifier 1. + pub const PHYSID1: Self = C22(0x02); + /// PHY identifier 2. + pub const PHYSID2: Self = C22(0x03); + /// Auto-negotiation advertisement. + pub const ADVERTISE: Self = C22(0x04); + /// Auto-negotiation link partner base page ability. + pub const LPA: Self = C22(0x05); + /// Auto-negotiation expansion. + pub const EXPANSION: Self = C22(0x06); + /// Auto-negotiation next page transmit. + pub const NEXT_PAGE_TRANSMIT: Self = C22(0x07); + /// Auto-negotiation link partner received next page. + pub const LP_RECEIVED_NEXT_PAGE: Self = C22(0x08); + /// Master-slave control. + pub const MASTER_SLAVE_CONTROL: Self = C22(0x09); + /// Master-slave status. + pub const MASTER_SLAVE_STATUS: Self = C22(0x0a); + /// PSE Control. + pub const PSE_CONTROL: Self = C22(0x0b); + /// PSE Status. + pub const PSE_STATUS: Self = C22(0x0c); + /// MMD Register control. + pub const MMD_CONTROL: Self = C22(0x0d); + /// MMD Register address data. + pub const MMD_DATA: Self = C22(0x0e); + /// Extended status. + pub const EXTENDED_STATUS: Self = C22(0x0f); + + /// Creates a new instance of `C22` with a vendor specific register. + pub const fn vendor_specific() -> Self { + build_assert!( + N > 0x0f && N < 0x20, + "Vendor-specific register address must be between 16 and 31" + ); + C22(N) + } +} + +impl private::Sealed for C22 {} + +impl Register for C22 { + fn read(&self, dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer + // `phydev`. + let ret = unsafe { + bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, self.0.into()) + }; + to_result(ret)?; + Ok(ret as u16) + } + + fn write(&self, dev: &mut Device, val: u16) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer + // `phydev`. + to_result(unsafe { + bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, self.0.into(), val) + }) + } + + fn read_status(dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_read_status(phydev) }; + to_result(ret)?; + Ok(ret as u16) + } +} + +/// A single MDIO clause 45 register device and address. +#[derive(Copy, Clone, Debug)] +pub struct Mmd(u8); + +impl Mmd { + /// Physical Medium Attachment/Dependent. + pub const PMAPMD: Self = Mmd(uapi::MDIO_MMD_PMAPMD as u8); + /// WAN interface sublayer. + pub const WIS: Self = Mmd(uapi::MDIO_MMD_WIS as u8); + /// Physical coding sublayer. + pub const PCS: Self = Mmd(uapi::MDIO_MMD_PCS as u8); + /// PHY Extender sublayer. + pub const PHYXS: Self = Mmd(uapi::MDIO_MMD_PHYXS as u8); + /// DTE Extender sublayer. + pub const DTEXS: Self = Mmd(uapi::MDIO_MMD_DTEXS as u8); + /// Transmission convergence. + pub const TC: Self = Mmd(uapi::MDIO_MMD_TC as u8); + /// Auto negotiation. + pub const AN: Self = Mmd(uapi::MDIO_MMD_AN as u8); + /// Separated PMA (1). + pub const SEPARATED_PMA1: Self = Mmd(8); + /// Separated PMA (2). + pub const SEPARATED_PMA2: Self = Mmd(9); + /// Separated PMA (3). + pub const SEPARATED_PMA3: Self = Mmd(10); + /// Separated PMA (4). + pub const SEPARATED_PMA4: Self = Mmd(11); + /// OFDM PMA/PMD. + pub const OFDM_PMAPMD: Self = Mmd(12); + /// Power unit. + pub const POWER_UNIT: Self = Mmd(13); + /// Clause 22 extension. + pub const C22_EXT: Self = Mmd(uapi::MDIO_MMD_C22EXT as u8); + /// Vendor specific 1. + pub const VEND1: Self = Mmd(uapi::MDIO_MMD_VEND1 as u8); + /// Vendor specific 2. + pub const VEND2: Self = Mmd(uapi::MDIO_MMD_VEND2 as u8); +} + +/// A single MDIO clause 45 register device and address. +/// +/// Clause 45 uses a 5-bit device address to access a specific MMD within +/// a port, then a 16-bit register address to access a location within +/// that device. `C45` represents this by storing a [`Mmd`] and +/// a register number. +pub struct C45 { + devad: Mmd, + regnum: u16, +} + +impl C45 { + /// Creates a new instance of `C45`. + pub fn new(devad: Mmd, regnum: u16) -> Self { + Self { devad, regnum } + } +} + +impl private::Sealed for C45 {} + +impl Register for C45 { + fn read(&self, dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call. + let ret = + unsafe { bindings::phy_read_mmd(phydev, self.devad.0.into(), self.regnum.into()) }; + to_result(ret)?; + Ok(ret as u16) + } + + fn write(&self, dev: &mut Device, val: u16) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Device`. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_write_mmd(phydev, self.devad.0.into(), self.regnum.into(), val) + }) + } + + fn read_status(dev: &mut Device) -> Result { + let phydev = dev.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_c45_read_status(phydev) }; + to_result(ret)?; + Ok(ret as u16) + } +} diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs new file mode 100644 index 000000000..de589792a --- /dev/null +++ b/rust/kernel/num.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Additional numerical features for the kernel. + +use core::ops; + +pub mod bounded; +pub mod casts; + +pub use bounded::*; + +/// Designates unsigned primitive types. +pub enum Unsigned {} + +/// Designates signed primitive types. +pub enum Signed {} + +mod private { + pub trait Sealed {} +} + +/// Describes core properties of integer types. +pub trait Integer: + private::Sealed + + Sized + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + ops::Add + + ops::AddAssign + + ops::Sub + + ops::SubAssign + + ops::Mul + + ops::MulAssign + + ops::Div + + ops::DivAssign + + ops::Rem + + ops::RemAssign + + ops::BitAnd + + ops::BitAndAssign + + ops::BitOr + + ops::BitOrAssign + + ops::BitXor + + ops::BitXorAssign + + ops::Shl + + ops::ShlAssign + + ops::Shr + + ops::ShrAssign + + ops::Not +{ + /// Whether this type is [`Signed`] or [`Unsigned`]. + type Signedness; + + /// Number of bits used for value representation. + const BITS: u32; +} + +macro_rules! impl_integer { + ($($type:ty: $signedness:ty), *) => { + $( + impl private::Sealed for $type {} + + impl Integer for $type { + type Signedness = $signedness; + + const BITS: u32 = <$type>::BITS; + } + )* + }; +} + +impl_integer!( + u8: Unsigned, + u16: Unsigned, + u32: Unsigned, + u64: Unsigned, + u128: Unsigned, + usize: Unsigned, + i8: Signed, + i16: Signed, + i32: Signed, + i64: Signed, + i128: Signed, + isize: Signed +); diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs new file mode 100644 index 000000000..2a2b0a4bc --- /dev/null +++ b/rust/kernel/num/bounded.rs @@ -0,0 +1,1165 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Bounded`], a wrapper around integer types limiting the number of bits +//! usable for value representation. + +use core::{ + cmp, + fmt, + ops::{ + self, + Deref, // + }, //, +}; + +use kernel::{ + num::{ + Integer, + Unsigned, // + }, + prelude::*, // +}; + +/// Evaluates to `true` if `$value` can be represented using at most `$n` bits in a `$type`. +/// +/// `expr` must be of type `type`, or the result will be incorrect. +/// +/// Can be used in const context. +macro_rules! fits_within { + ($value:expr, $type:ty, $n:expr) => {{ + let shift: u32 = <$type>::BITS - $n; + + // `value` fits within `$n` bits if shifting it left by the number of unused bits, then + // right by the same number, doesn't change it. + // + // This method has the benefit of working for both unsigned and signed values. + ($value << shift) >> shift == $value + }}; +} + +/// Returns `true` if `value` can be represented with at most `N` bits in a `T`. +#[inline(always)] +fn fits_within(value: T, num_bits: u32) -> bool { + fits_within!(value, T, num_bits) +} + +/// An integer value that requires only the `N` least significant bits of the wrapped type to be +/// encoded. +/// +/// This limits the number of usable bits in the wrapped integer type, and thus the stored value to +/// a narrower range, which provides guarantees that can be useful when working within e.g. +/// bitfields. +/// +/// # Invariants +/// +/// - `N` is greater than `0`. +/// - `N` is less than or equal to `T::BITS`. +/// - Stored values can be represented with at most `N` bits. +/// +/// # Examples +/// +/// The preferred way to create values is through constants and the [`Bounded::new`] family of +/// constructors, as they trigger a build error if the type invariants cannot be upheld. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // An unsigned 8-bit integer, of which only the 4 LSBs are used. +/// // The value `15` is statically validated to fit that constraint at build time. +/// let v = Bounded::::new::<15>(); +/// assert_eq!(v.get(), 15); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-8>(); +/// assert_eq!(v.get(), -8); +/// +/// // This doesn't build: a `u8` is smaller than the requested 9 bits. +/// // let _ = Bounded::::new::<10>(); +/// +/// // This also doesn't build: the requested value doesn't fit within 4 signed bits. +/// // let _ = Bounded::::new::<8>(); +/// ``` +/// +/// Values can also be validated at runtime with [`Bounded::try_new`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This succeeds because `15` can be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(15).is_some()); +/// +/// // This fails because `16` cannot be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(16).is_none()); +/// ``` +/// +/// Non-constant expressions can be validated at build-time thanks to compiler optimizations. This +/// should be used with caution, on simple expressions only. +/// +/// ``` +/// use kernel::num::Bounded; +/// # fn some_number() -> u32 { 0xffffffff } +/// +/// // Here the compiler can infer from the mask that the type invariants are not violated, even +/// // though the value returned by `some_number` is not statically known. +/// let v = Bounded::::from_expr(some_number() & 0xf); +/// ``` +/// +/// Comparison and arithmetic operations are supported on [`Bounded`]s with a compatible backing +/// type, regardless of their number of valid bits. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v1 = Bounded::::new::<4>(); +/// let v2 = Bounded::::new::<15>(); +/// +/// assert!(v1 != v2); +/// assert!(v1 < v2); +/// assert_eq!(v1 + v2, 19); +/// assert_eq!(v2 % v1, 3); +/// ``` +/// +/// These operations are also supported between a [`Bounded`] and its backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<15>(); +/// +/// assert!(v == 15); +/// assert!(v > 12); +/// assert_eq!(v + 5, 20); +/// assert_eq!(v / 3, 5); +/// ``` +/// +/// A change of backing types is possible using [`Bounded::cast`], and the number of valid bits can +/// be extended or reduced with [`Bounded::extend`] and [`Bounded::try_shrink`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<127>(); +/// +/// // Changes backing type from `u32` to `u16`. +/// let _: Bounded = v.cast(); +/// +/// // This does not build, as `u8` is smaller than 12 bits. +/// // let _: Bounded = v.cast(); +/// +/// // We can safely extend the number of bits... +/// let _ = v.extend::<15>(); +/// +/// // ... to the limits of the backing type. This doesn't build as a `u32` cannot contain 33 bits. +/// // let _ = v.extend::<33>(); +/// +/// // Reducing the number of bits is validated at runtime. This works because `127` can be +/// // represented with 8 bits. +/// assert!(v.try_shrink::<8>().is_some()); +/// +/// // ... but not with 6, so this fails. +/// assert!(v.try_shrink::<6>().is_none()); +/// ``` +/// +/// Infallible conversions from a primitive integer to a large-enough [`Bounded`] are supported. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This unsigned `Bounded` has 8 bits, so it can represent any `u8`. +/// let v = Bounded::::from(128u8); +/// assert_eq!(v.get(), 128); +/// +/// // This signed `Bounded` has 8 bits, so it can represent any `i8`. +/// let v = Bounded::::from(-128i8); +/// assert_eq!(v.get(), -128); +/// +/// // This doesn't build, as this 6-bit `Bounded` does not have enough capacity to represent a +/// // `u8` (regardless of the passed value). +/// // let _ = Bounded::::from(10u8); +/// +/// // Booleans can be converted into unsigned `Bounded`s. +/// +/// let v = Bounded::::from(false); +/// assert_eq!(v.get(), 0); +/// +/// let v = Bounded::::from(true); +/// assert_eq!(v.get(), 1); +/// +/// // This does not build because `i8` is signed. +/// // let _ = Bounded::::from(true); +/// ``` +/// +/// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and +/// dependent on the number of bits used for value representation, not on the backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // Even though its backing type is `u32`, this `Bounded` only uses 6 bits and thus can safely +/// // be converted to a `u8`. +/// let v = Bounded::::new::<63>(); +/// assert_eq!(u8::from(v), 63); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-128>(); +/// assert_eq!(i8::from(v), -128); +/// +/// // This however does not build, as 10 bits won't fit into a `u8` (regardless of the actually +/// // contained value). +/// let _v = Bounded::::new::<10>(); +/// // assert_eq!(u8::from(_v), 10); +/// +/// // Unsigned single-bit `Bounded`s can be converted into a boolean. +/// let v = Bounded::::new::<1>(); +/// assert_eq!(bool::from(v), true); +/// +/// let v = Bounded::::new::<0>(); +/// assert_eq!(bool::from(v), false); +/// +/// // This does not build because `i8` is signed. +/// // let v = Bounded::::new::<-1>(); +/// // let _ = bool::from(v); +/// ``` +/// +/// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the +/// [`TryIntoBounded`] trait. +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fit into 6 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v, None); +/// ``` +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Default, Hash)] +pub struct Bounded(T); + +/// Validating the value as a const expression cannot be done as a regular method, as the +/// arithmetic operations we rely on to check the bounds are not const. Thus, implement +/// [`Bounded::new`] using a macro. +macro_rules! impl_const_new { + ($($type:ty)*) => { + $( + impl Bounded<$type, N> { + /// Creates a [`Bounded`] for the constant `VALUE`. + /// + /// Fails at build time if `VALUE` cannot be represented with `N` bits. + /// + /// This method should be preferred to [`Self::from_expr`] whenever possible. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + #[doc = ::core::concat!( + "let v = Bounded::<", + ::core::stringify!($type), + ", 4>::new::<7>();")] + /// assert_eq!(v.get(), 7); + /// ``` + pub const fn new() -> Self { + // Statically assert that `VALUE` fits within the set number of bits. + const_assert!(fits_within!(VALUE, $type, N)); + + // SAFETY: `fits_within` confirmed that `VALUE` can be represented within + // `N` bits. + unsafe { Self::__new(VALUE) } + } + } + )* + }; +} + +impl_const_new!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +impl Bounded +where + T: Integer, +{ + /// Private constructor enforcing the type invariants. + /// + /// All instances of [`Bounded`] must be created through this method as it enforces most of the + /// type invariants. + /// + /// # Safety + /// + /// The caller must ensure that `value` can be represented within `N` bits. + const unsafe fn __new(value: T) -> Self { + // Enforce the type invariants. + // `N` cannot be zero. + const_assert!(N != 0); + // The backing type is at least as large as `N` bits. + const_assert!(N <= T::BITS); + + // INVARIANT: The caller ensures `value` fits within `N` bits. + Self(value) + } + + /// Attempts to turn `value` into a `Bounded` using `N` bits. + /// + /// Returns [`None`] if `value` doesn't fit within `N` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::try_new(1); + /// assert_eq!(v.as_deref().copied(), Some(1)); + /// + /// let v = Bounded::::try_new(-2); + /// assert_eq!(v.as_deref().copied(), Some(-2)); + /// + /// // `0x1ff` doesn't fit into 8 unsigned bits. + /// let v = Bounded::::try_new(0x1ff); + /// assert_eq!(v, None); + /// + /// // The range of values representable with 4 bits is `[-8..=7]`. The following tests these + /// // limits. + /// let v = Bounded::::try_new(-8); + /// assert_eq!(v.map(Bounded::get), Some(-8)); + /// let v = Bounded::::try_new(-9); + /// assert_eq!(v, None); + /// let v = Bounded::::try_new(7); + /// assert_eq!(v.map(Bounded::get), Some(7)); + /// let v = Bounded::::try_new(8); + /// assert_eq!(v, None); + /// ``` + pub fn try_new(value: T) -> Option { + fits_within(value, N).then(|| { + // SAFETY: `fits_within` confirmed that `value` can be represented within `N` bits. + unsafe { Self::__new(value) } + }) + } + + /// Checks that `expr` is valid for this type at compile-time and build a new value. + /// + /// This relies on [`build_assert!`] and guaranteed optimization to perform validation at + /// compile-time. If `expr` cannot be proved to be within the requested bounds at compile-time, + /// use the fallible [`Self::try_new`] instead. + /// + /// Limit this to simple, easily provable expressions, and prefer one of the [`Self::new`] + /// constructors whenever possible as they statically validate the value instead of relying on + /// compiler optimizations. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// # fn some_number() -> u32 { 0xffffffff } + /// + /// // Some undefined number. + /// let v: u32 = some_number(); + /// + /// // Triggers a build error as `v` cannot be asserted to fit within 4 bits... + /// // let _ = Bounded::::from_expr(v); + /// + /// // ... but this works as the compiler can assert the range from the mask. + /// let _ = Bounded::::from_expr(v & 0xf); + /// + /// // These expressions are simple enough to be proven correct, but since they are static the + /// // `new` constructor should be preferred. + /// assert_eq!(Bounded::::from_expr(1).get(), 1); + /// assert_eq!(Bounded::::from_expr(0xff).get(), 0xff); + /// ``` + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub fn from_expr(expr: T) -> Self { + crate::build_assert::build_assert!( + fits_within(expr, N), + "Requested value larger than maximal representable value." + ); + + // SAFETY: `fits_within` confirmed that `expr` can be represented within `N` bits. + unsafe { Self::__new(expr) } + } + + /// Returns the wrapped value as the backing type. + /// + /// This is similar to the [`Deref`] implementation, but doesn't enforce the size invariant of + /// the [`Bounded`], which might produce slightly less optimal code. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// assert_eq!(v.get(), 7u32); + /// ``` + pub const fn get(self) -> T { + self.0 + } + + /// Increases the number of bits usable for `self`. + /// + /// This operation cannot fail. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// let larger_v = v.extend::<12>(); + /// // The contained values are equal even though `larger_v` has a bigger capacity. + /// assert_eq!(larger_v, v); + /// ``` + pub const fn extend(self) -> Bounded { + const_assert!( + M >= N, + "Requested number of bits is less than the current representation." + ); + + // SAFETY: The value did fit within `N` bits, so it will all the more fit within + // the larger `M` bits. + unsafe { Bounded::__new(self.0) } + } + + /// Attempts to shrink the number of bits usable for `self`. + /// + /// Returns [`None`] if the value of `self` cannot be represented within `M` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// + /// // `7` can be represented using 3 unsigned bits... + /// let smaller_v = v.try_shrink::<3>(); + /// assert_eq!(smaller_v.as_deref().copied(), Some(7)); + /// + /// // ... but doesn't fit within `2` bits. + /// assert_eq!(v.try_shrink::<2>(), None); + /// ``` + pub fn try_shrink(self) -> Option> { + Bounded::::try_new(self.get()) + } + + /// Casts `self` into a [`Bounded`] backed by a different storage type, but using the same + /// number of valid bits. + /// + /// Both `T` and `U` must be of same signedness, and `U` must be at least as large as + /// `N` bits, or a build error will occur. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<127>(); + /// + /// let u16_v: Bounded = v.cast(); + /// assert_eq!(u16_v.get(), 127); + /// + /// // This won't build: a `u8` is smaller than the required 12 bits. + /// // let _: Bounded = v.cast(); + /// ``` + pub fn cast(self) -> Bounded + where + U: TryFrom + Integer, + T: Integer, + U: Integer, + { + // SAFETY: The converted value is represented using `N` bits, `U` can contain `N` bits, and + // `U` and `T` have the same sign, hence this conversion cannot fail. + let value = unsafe { U::try_from(self.get()).unwrap_unchecked() }; + + // SAFETY: Although the backing type has changed, the value is still represented within + // `N` bits, and with the same signedness. + unsafe { Bounded::__new(value) } + } + + /// Right-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N - SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff00>(); + /// let v_shifted: Bounded:: = v.shr::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff); + /// ``` + pub fn shr(self) -> Bounded { + const_assert!(SHIFT < T::BITS); + const_assert!(RES + SHIFT >= N); + + // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. + unsafe { Bounded::__new(self.0 >> SHIFT) } + } + + /// Right-shifts `self` by `SHIFT` if that loses no set bits, and returns the result as a + /// `Bounded<_, RES>`, where `RES >= N - SHIFT`. + /// + /// Returns [`None`] if any of the `SHIFT` least significant bits of `self` is set. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff00>(); + /// let v_shifted: Option> = v.shr_exact::<8, _>(); + /// + /// assert_eq!(v_shifted.map(|v| v.get()), Some(0xff)); + /// + /// // A set bit would be shifted out. + /// let v = Bounded::::new::<0xff01>(); + /// let v_shifted: Option> = v.shr_exact::<8, _>(); + /// + /// assert!(v_shifted.is_none()); + /// ``` + #[inline] + pub fn shr_exact(self) -> Option> { + let shifted = self.shr::(); + if shifted.get() << SHIFT == self.0 { + Some(shifted) + } else { + None + } + } + + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N + SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff>(); + /// let v_shifted: Bounded:: = v.shl::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff00); + /// ``` + pub fn shl(self) -> Bounded { + const_assert!(RES >= N + SHIFT); + + // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. + unsafe { Bounded::__new(self.0 << SHIFT) } + } +} + +impl Deref for Bounded +where + T: Integer, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + // Enforce the invariant to inform the compiler of the bounds of the value. + if !fits_within(self.0, N) { + // SAFETY: Per the `Bounded` invariants, `fits_within` can never return `false` on the + // value of a valid instance. + unsafe { core::hint::unreachable_unchecked() } + } + + &self.0 + } +} + +/// Trait similar to [`TryInto`] but for [`Bounded`], to avoid conflicting implementations. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fit into 6 bits. +/// let v: Option> = 128u32.try_into_bounded(); +/// assert_eq!(v, None); +/// ``` +pub trait TryIntoBounded { + /// Attempts to convert `self` into a [`Bounded`] using `N` bits. + /// + /// Returns [`None`] if `self` does not fit into the target type. + fn try_into_bounded(self) -> Option>; +} + +/// Any integer value can be attempted to be converted into a [`Bounded`] of any size. +impl TryIntoBounded for U +where + T: Integer, + U: TryInto, +{ + fn try_into_bounded(self) -> Option> { + self.try_into().ok().and_then(Bounded::try_new) + } +} + +// Comparisons between `Bounded`s. + +impl PartialEq> for Bounded +where + T: Integer, + U: Integer, + T: PartialEq, +{ + fn eq(&self, other: &Bounded) -> bool { + self.get() == other.get() + } +} + +impl Eq for Bounded where T: Integer {} + +impl PartialOrd> for Bounded +where + T: Integer, + U: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &Bounded) -> Option { + self.get().partial_cmp(&other.get()) + } +} + +impl Ord for Bounded +where + T: Integer, + T: Ord, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.get().cmp(&other.get()) + } +} + +// Comparisons between a `Bounded` and its backing type. + +impl PartialEq for Bounded +where + T: Integer, + T: PartialEq, +{ + fn eq(&self, other: &T) -> bool { + self.get() == *other + } +} + +impl PartialOrd for Bounded +where + T: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &T) -> Option { + self.get().partial_cmp(other) + } +} + +// Implementations of `core::ops` for two `Bounded` with the same backing type. + +impl ops::Add> for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: Bounded) -> Self::Output { + self.get() + rhs.get() + } +} + +impl ops::BitAnd> for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: Bounded) -> Self::Output { + self.get() & rhs.get() + } +} + +impl ops::BitOr> for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: Bounded) -> Self::Output { + self.get() | rhs.get() + } +} + +impl ops::BitXor> for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: Bounded) -> Self::Output { + self.get() ^ rhs.get() + } +} + +impl ops::Div> for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: Bounded) -> Self::Output { + self.get() / rhs.get() + } +} + +impl ops::Mul> for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: Bounded) -> Self::Output { + self.get() * rhs.get() + } +} + +impl ops::Rem> for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: Bounded) -> Self::Output { + self.get() % rhs.get() + } +} + +impl ops::Sub> for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: Bounded) -> Self::Output { + self.get() - rhs.get() + } +} + +// Implementations of `core::ops` between a `Bounded` and its backing type. + +impl ops::Add for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: T) -> Self::Output { + self.get() + rhs + } +} + +impl ops::BitAnd for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: T) -> Self::Output { + self.get() & rhs + } +} + +impl ops::BitOr for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: T) -> Self::Output { + self.get() | rhs + } +} + +impl ops::BitXor for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: T) -> Self::Output { + self.get() ^ rhs + } +} + +impl ops::Div for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: T) -> Self::Output { + self.get() / rhs + } +} + +impl ops::Mul for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: T) -> Self::Output { + self.get() * rhs + } +} + +impl ops::Neg for Bounded +where + T: Integer, + T: ops::Neg, +{ + type Output = T; + + fn neg(self) -> Self::Output { + -self.get() + } +} + +impl ops::Not for Bounded +where + T: Integer, + T: ops::Not, +{ + type Output = T; + + fn not(self) -> Self::Output { + !self.get() + } +} + +impl ops::Rem for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: T) -> Self::Output { + self.get() % rhs + } +} + +impl ops::Sub for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: T) -> Self::Output { + self.get() - rhs + } +} + +// Proxy implementations of `core::fmt`. + +impl fmt::Display for Bounded +where + T: Integer, + T: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Binary for Bounded +where + T: Integer, + T: fmt::Binary, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerExp for Bounded +where + T: Integer, + T: fmt::LowerExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerHex for Bounded +where + T: Integer, + T: fmt::LowerHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Octal for Bounded +where + T: Integer, + T: fmt::Octal, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperExp for Bounded +where + T: Integer, + T: fmt::UpperExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperHex for Bounded +where + T: Integer, + T: fmt::UpperHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +/// Implements `$trait` for all [`Bounded`] types represented using `$num_bits`. +/// +/// This is used to declare size properties as traits that we can constrain against in impl blocks. +macro_rules! impl_size_rule { + ($trait:ty, $($num_bits:literal)*) => { + $( + impl $trait for Bounded where T: Integer {} + )* + }; +} + +/// Local trait expressing the fact that a given [`Bounded`] has at least `N` bits used for value +/// representation. +trait AtLeastXBits {} + +/// Implementations for infallibly converting a primitive type into a [`Bounded`] that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod atleast_impls { + use super::*; + + // Number of bits at least as large as 64. + impl_size_rule!(AtLeastXBits<64>, 64); + + // Anything 64 bits or more is also larger than 32. + impl AtLeastXBits<32> for T where T: AtLeastXBits<64> {} + // Other numbers of bits at least as large as 32. + impl_size_rule!(AtLeastXBits<32>, + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + ); + + // Anything 32 bits or more is also larger than 16. + impl AtLeastXBits<16> for T where T: AtLeastXBits<32> {} + // Other numbers of bits at least as large as 16. + impl_size_rule!(AtLeastXBits<16>, + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + ); + + // Anything 16 bits or more is also larger than 8. + impl AtLeastXBits<8> for T where T: AtLeastXBits<16> {} + // Other numbers of bits at least as large as 8. + impl_size_rule!(AtLeastXBits<8>, 8 9 10 11 12 13 14 15); +} + +/// Generates `From` implementations from a primitive type into a [`Bounded`] with +/// enough bits to store any value of that type. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of [`AtLeastXBits`]'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_from_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`", + ::core::stringify!($type), + "`] into a [`Bounded`] of same signedness with enough bits to store it.")] + impl From<$type> for Bounded + where + $type: Integer, + T: Integer::Signedness> + From<$type>, + Self: AtLeastXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: $type) -> Self { + // SAFETY: The trait bound on `Self` guarantees that `N` bits is + // enough to hold any value of the source type. + unsafe { Self::__new(T::from(value)) } + } + } + )* + } +} + +impl_from_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +/// Local trait expressing the fact that a given [`Bounded`] fits into a primitive type of `N` bits, +/// provided they have the same signedness. +trait FitsInXBits {} + +/// Implementations for infallibly converting a [`Bounded`] into a primitive type that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod fits_impls { + use super::*; + + // Number of bits that fit into a 8-bits primitive. + impl_size_rule!(FitsInXBits<8>, 1 2 3 4 5 6 7 8); + + // Anything that fits into 8 bits also fits into 16. + impl FitsInXBits<16> for T where T: FitsInXBits<8> {} + // Other number of bits that fit into a 16-bits primitive. + impl_size_rule!(FitsInXBits<16>, 9 10 11 12 13 14 15 16); + + // Anything that fits into 16 bits also fits into 32. + impl FitsInXBits<32> for T where T: FitsInXBits<16> {} + // Other number of bits that fit into a 32-bits primitive. + impl_size_rule!(FitsInXBits<32>, + 17 18 19 20 21 22 23 24 + 25 26 27 28 29 30 31 32 + ); + + // Anything that fits into 32 bits also fits into 64. + impl FitsInXBits<64> for T where T: FitsInXBits<32> {} + // Other number of bits that fit into a 64-bits primitive. + impl_size_rule!(FitsInXBits<64>, + 33 34 35 36 37 38 39 40 + 41 42 43 44 45 46 47 48 + 49 50 51 52 53 54 55 56 + 57 58 59 60 61 62 63 64 + ); +} + +/// Generates [`From`] implementations from a [`Bounded`] into a primitive type that is +/// guaranteed to contain it. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of `AtLeastXBits`'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_into_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`Bounded`] with no more bits than a [`", + ::core::stringify!($type), + "`] and of same signedness into [`", + ::core::stringify!($type), + "`]")] + impl From> for $type + where + $type: Integer + TryFrom, + T: Integer::Signedness>, + Bounded: FitsInXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: Bounded) -> $type { + // SAFETY: The trait bound on `Bounded` ensures that any value it holds (which + // is constrained to `N` bits) can fit into the destination type, so this + // conversion cannot fail. + unsafe { <$type>::try_from(value.get()).unwrap_unchecked() } + } + } + )* + } +} + +impl_into_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +// Unsigned single-bit `Bounded`s can be converted to a boolean. + +impl From> for bool +where + T: Integer + Zeroable, +{ + fn from(value: Bounded) -> Self { + value.get() != Zeroable::zeroed() + } +} + +// Booleans can be converted to unsigned `Bounded`s. + +impl From for Bounded +where + T: Integer + From, +{ + fn from(value: bool) -> Self { + // SAFETY: A boolean is represented by `0` or `1`, so it fits within any valid unsigned + // `Bounded` width. + unsafe { Self::__new(T::from(value)) } + } +} + +impl Bounded +where + T: Integer + Zeroable, +{ + /// Converts this [`Bounded`] into a [`bool`]. + /// + /// This is a shorter way of writing `bool::from(self)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// assert_eq!(Bounded::::new::<0>().into_bool(), false); + /// assert_eq!(Bounded::::new::<1>().into_bool(), true); + /// ``` + pub fn into_bool(self) -> bool { + self.into() + } +} diff --git a/rust/kernel/num/casts.rs b/rust/kernel/num/casts.rs new file mode 100644 index 000000000..7e6c7dec7 --- /dev/null +++ b/rust/kernel/num/casts.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helpers for performing lossless integer casts. +//! +//! The `as` keyword can be used to perform casts between integer types, but it unfortunately makes +//! no distinction between casts that are lossless, and casts from a larger type into a smaller one +//! that might silently strip data away. Thus, its use in the kernel is discouraged in favor of +//! [`From`] implementations. +//! +//! Conversely, there are casts that are lossless depending on the build architecture (such as +//! casting [`usize`] to [`u64`] on 32 or 64 bit archs), but not supported by [`From`] +//! implementations in the standard library because they are not portable. It does however make +//! sense for the kernel to support these, if only for code that is architecture-specific. +//! +//! This module provides ways to perform such conversions safely: +//! +//! - A series of const functions (e.g. [`usize_as_u64`]) supporting safe conversions in const +//! context. Conversions supported by [`From`] implementations in the standard library are also +//! covered as the [`From`] trait cannot be used in const context. +//! - Two extension traits, [`FromSafeCast`] and [`IntoSafeCast`], providing conversion methods +//! similar to [`From`] and [`Into`] for conversions that are safe to perform in the kernel, but +//! not supported by the standard library. +//! - Another series of const functions (e.g. [`u64_into_u8`]) supporting the conversion of a const +//! value from a larger type into a smaller one, provided the value fits into the destination +//! type. This is useful if a constant is defined as a larger type, but needs to be used as a +//! smaller one. +//! - An [`arch`] sub-module, defining more conversion functions that are only guaranteed to be +//! lossless for a given pointer size. These can only be used in code that is specific to a +//! given pointer size. +//! +//! # Examples +//! +//! ``` +//! use kernel::num::casts::{self, FromSafeCast, IntoSafeCast}; +//! +//! // Conversion from const context. +//! const USIZED_CONST: usize = casts::u8_as_usize(255u8); +//! +//! // Non-const conversions. +//! let a = u64::from_safe_cast(4096usize); +//! let b: u64 = 4096usize.into_safe_cast(); +//! ``` + +use crate::prelude::*; + +/// Implements safe `as` conversion functions from a given type into a series of target types. +/// +/// These functions can be used in place of `as`, with the guarantee that they will be lossless. +macro_rules! impl_safe_as { + ($from:ty as { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Losslessly converts a [`", + ::core::stringify!($from), + "`] into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This conversion is allowed as it is always lossless. Prefer this over the `as` + /// keyword to ensure no lossy casts are performed. + /// + /// This is for use from a `const` context. For non `const` use, prefer the + /// [`FromSafeCast`] and [`IntoSafeCast`] traits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_as_", + ::core::stringify!($into), + "(1", + ::core::stringify!($from), + "), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _as_ $into>](value: $from) -> $into { + $crate::static_assert!(size_of::<$into>() >= size_of::<$from>()); + + value as $into + } + } + )* + }; +} + +// Valid `Into` transformations. +impl_safe_as!(u8 as { u16, u32, u64, usize }); +impl_safe_as!(u16 as { u32, u64, usize }); +impl_safe_as!(u32 as { u64 }); +// A `usize` fits into a `u64` on all supported platforms. +impl_safe_as!(usize as { u64 }); +// A `u32` fits into a `usize` on all supported platforms. +impl_safe_as!(u32 as { usize }); + +/// Extension trait providing guaranteed lossless cast to [`Self`] from `T`. +/// +/// The standard library's [`From`] implementations do not cover conversions that are not portable +/// or future-proof. For instance, even though it is safe today, [`From`] is not implemented +/// for [`u64`] because of the possibility of needing to support larger-than-64bit architectures in +/// the future. +/// +/// The workaround is to either deal with the error handling of [`TryFrom`] for an operation that +/// technically cannot fail, or to use the `as` keyword, which can silently strip data if the +/// destination type is smaller than the source. +/// +/// Both options are hardly acceptable for the kernel. It is also a much more architecture +/// dependent environment, supporting only 32 and 64 bit architectures, with some modules +/// explicitly depending on a specific bus width that could greatly benefit from infallible +/// conversion operations. +/// +/// Thus this extension trait that provides, for all architectures supported by the kernel, +/// conversion methods between types for which such a cast is lossless. +/// +/// In other words, this trait is implemented if, for all supported targets and with `t: T`, the +/// `t as Self` operation is completely lossless. +/// +/// Prefer this over the `as` keyword to guarantee that no lossy casts are performed. +/// +/// If you need to perform a conversion in `const` context, use [`u32_as_usize`], [`usize_as_u64`], +/// etc. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::FromSafeCast; +/// +/// assert_eq!(usize::from_safe_cast(0xf00u32), 0xf00usize); +/// ``` +pub trait FromSafeCast { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast(value: T) -> Self; +} + +// A `usize` fits into a `u64` on all supported platforms. +impl FromSafeCast for u64 { + #[inline] + fn from_safe_cast(value: usize) -> Self { + usize_as_u64(value) + } +} + +// A `u32` fits into a `usize` on all supported platforms. +impl FromSafeCast for usize { + #[inline] + fn from_safe_cast(value: u32) -> Self { + u32_as_usize(value) + } +} + +/// Counterpart to the [`FromSafeCast`] trait, i.e. this trait is to [`FromSafeCast`] what [`Into`] +/// is to [`From`]. +/// +/// See the documentation of [`FromSafeCast`] for the motivation. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::IntoSafeCast; +/// +/// assert_eq!(0xf00usize, 0xf00u32.into_safe_cast()); +/// ``` +pub trait IntoSafeCast { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast(self) -> T; +} + +/// Reverse operation for types implementing [`FromSafeCast`]. +impl IntoSafeCast for S +where + T: FromSafeCast, +{ + #[inline] + fn into_safe_cast(self) -> T { + T::from_safe_cast(self) + } +} + +/// Implements lossless conversion of a constant from a larger type into a smaller one. +macro_rules! impl_const_into { + ($from:ty => { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Performs a build-time safe conversion of a [`", + ::core::stringify!($from), + "`] constant value into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This checks at compile-time that the conversion is lossless, and triggers a build + /// error if it isn't. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + /// // Succeeds because the value of the source fits into the destination's type. + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_into_", + ::core::stringify!($into), + "::<1", + ::core::stringify!($from), + ">(), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _into_ $into>]() -> $into { + // Make sure that the target type is smaller than the source one. + $crate::static_assert!($from::BITS >= $into::BITS); + // CAST: we statically enforced above that `$from` is larger than `$into`, so the + // `as` conversion will be lossless. + $crate::const_assert!(N >= $into::MIN as $from && N <= $into::MAX as $from); + + N as $into + } + } + )* + }; +} + +impl_const_into!(usize => { u8, u16, u32 }); +impl_const_into!(u64 => { u8, u16, u32 }); +impl_const_into!(u32 => { u8, u16 }); +impl_const_into!(u16 => { u8 }); + +/// Conversions that are only lossless for the current architecture. +/// +/// # Portability +/// +/// Callers of this module become dependent on the setting of `CONFIG_64BIT`. Use with caution, and +/// never in code that is portable across pointer sizes. +pub mod arch { + /// Trait identical to [`FromSafeCast`](super::FromSafeCast), but for conversions that are not + /// available on all architectures. + pub trait FromSafeCastArch { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast_arch(value: T) -> Self; + } + + /// Trait identical to [`IntoSafeCast`](super::IntoSafeCast), but for conversions that are not + /// available on all architectures. + pub trait IntoSafeCastArch { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast_arch(self) -> T; + } + + /// Reverse operation for types implementing [`FromSafeCastArch`]. + impl IntoSafeCastArch for S + where + T: FromSafeCastArch, + { + #[inline] + fn into_safe_cast_arch(self) -> T { + T::from_safe_cast_arch(self) + } + } + + /// A [`u64`] fits into a [`usize`] on 64-bit platforms. + #[cfg(CONFIG_64BIT)] + #[inline] + pub const fn u64_as_usize(value: u64) -> usize { + value as usize + } + + #[cfg(CONFIG_64BIT)] + impl FromSafeCastArch for usize { + #[inline] + fn from_safe_cast_arch(value: u64) -> Self { + u64_as_usize(value) + } + } + + /// A [`usize`] fits into a [`u32`] on 32-bit platforms. + #[cfg(not(CONFIG_64BIT))] + #[inline] + pub const fn usize_as_u32(value: usize) -> u32 { + value as u32 + } + + #[cfg(not(CONFIG_64BIT))] + impl FromSafeCastArch for u32 { + #[inline] + fn from_safe_cast_arch(value: usize) -> Self { + usize_as_u32(value) + } + } +} diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 000000000..d0318f62a --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Device Tree / Open Firmware abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for OF drivers. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// An open firmware device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::of_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` and +// does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); +} + +impl DeviceId { + /// Create a new device id from an OF 'compatible' string. + pub const fn new(compatible: &'static CStr) -> Self { + let src = compatible.to_bytes_with_nul(); + // Replace with `bindings::of_device_id::default()` once stabilized for `const`. + // SAFETY: FFI type is valid to be zero-initialized. + let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() }; + + // TODO: Use `copy_from_slice` once stabilized for `const`. + let mut i = 0; + while i < src.len() { + of.compatible[i] = src[i]; + i += 1; + } + + Self(of) + } +} + +/// Create an OF `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! of_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("of", $crate::of::DeviceId, $($tt)*); + }; +} diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs new file mode 100644 index 000000000..62e446761 --- /dev/null +++ b/rust/kernel/opp.rs @@ -0,0 +1,1148 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Operating performance points. +//! +//! This module provides rust abstractions for interacting with the OPP subsystem. +//! +//! C header: [`include/linux/pm_opp.h`](srctree/include/linux/pm_opp.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + cpumask::{Cpumask, CpumaskVar}, + device::Device, + error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + ffi::{c_char, c_ulong}, + prelude::*, + str::CString, + sync::aref::{ARef, AlwaysRefCounted}, + types::Opaque, +}; + +#[cfg(CONFIG_CPU_FREQ)] +/// Frequency table implementation. +mod freq { + use super::*; + use crate::cpufreq; + use core::ops::Deref; + + /// OPP frequency table. + /// + /// A [`cpufreq::Table`] created from [`Table`]. + pub struct FreqTable { + dev: ARef, + ptr: *mut bindings::cpufreq_frequency_table, + } + + impl FreqTable { + /// Creates a new instance of [`FreqTable`] from [`Table`]. + pub(crate) fn new(table: &Table) -> Result { + let mut ptr: *mut bindings::cpufreq_frequency_table = ptr::null_mut(); + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_init_cpufreq_table(table.dev.as_raw(), &mut ptr) + })?; + + Ok(Self { + dev: table.dev.clone(), + ptr, + }) + } + + /// Returns a reference to the underlying [`cpufreq::Table`]. + #[inline] + fn table(&self) -> &cpufreq::Table { + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { cpufreq::Table::from_raw(self.ptr) } + } + } + + impl Deref for FreqTable { + type Target = cpufreq::Table; + + #[inline] + fn deref(&self) -> &Self::Target { + self.table() + } + } + + impl Drop for FreqTable { + fn drop(&mut self) { + // SAFETY: The pointer was created via `dev_pm_opp_init_cpufreq_table`, and is only + // freed here. + unsafe { + bindings::dev_pm_opp_free_cpufreq_table(self.dev.as_raw(), &mut self.as_raw()) + }; + } + } +} + +#[cfg(CONFIG_CPU_FREQ)] +pub use freq::FreqTable; + +use core::{marker::PhantomData, ptr}; + +use macros::vtable; + +/// Creates a null-terminated slice of pointers to [`CString`]s. +fn to_c_str_array(names: &[CString]) -> Result> { + // Allocated a null-terminated vector of pointers. + let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?; + + for name in names.iter() { + list.push(name.as_char_ptr(), GFP_KERNEL)?; + } + + list.push(ptr::null(), GFP_KERNEL)?; + Ok(list) +} + +/// The voltage unit. +/// +/// Represents voltage in microvolts, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::opp::MicroVolt; +/// +/// let raw = 90500; +/// let volt = MicroVolt(raw); +/// +/// assert_eq!(usize::from(volt), raw); +/// assert_eq!(volt, MicroVolt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroVolt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(volt: MicroVolt) -> Self { + volt.0 + } +} + +/// The power unit. +/// +/// Represents power in microwatts, wrapping a [`c_ulong`] value. +/// +/// # Examples +/// +/// ``` +/// use kernel::opp::MicroWatt; +/// +/// let raw = 1000000; +/// let power = MicroWatt(raw); +/// +/// assert_eq!(usize::from(power), raw); +/// assert_eq!(power, MicroWatt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroWatt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(power: MicroWatt) -> Self { + power.0 + } +} + +/// Handle for a dynamically created [`OPP`]. +/// +/// The associated [`OPP`] is automatically removed when the [`Token`] is dropped. +/// +/// # Examples +/// +/// The following example demonstrates how to create an [`OPP`] dynamically. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::sync::aref::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +pub struct Token { + dev: ARef, + freq: Hertz, +} + +impl Token { + /// Dynamically adds an [`OPP`] and returns a [`Token`] that removes it on drop. + fn new(dev: &ARef, mut data: Data) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_add_dynamic(dev.as_raw(), &mut data.0) })?; + Ok(Self { + dev: dev.clone(), + freq: data.freq(), + }) + } +} + +impl Drop for Token { + fn drop(&mut self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_remove(self.dev.as_raw(), self.freq.into()) }; + } +} + +/// OPP data. +/// +/// Rust abstraction for the C `struct dev_pm_opp_data`, used to define operating performance +/// points (OPPs) dynamically. +/// +/// # Examples +/// +/// The following example demonstrates how to create an [`OPP`] with [`Data`]. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::sync::aref::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +#[repr(transparent)] +pub struct Data(bindings::dev_pm_opp_data); + +impl Data { + /// Creates a new instance of [`Data`]. + /// + /// This can be used to define a dynamic OPP to be added to a device. + pub fn new(freq: Hertz, volt: MicroVolt, level: u32, turbo: bool) -> Self { + Self(bindings::dev_pm_opp_data { + turbo, + freq: freq.into(), + u_volt: volt.into(), + level, + }) + } + + /// Adds an [`OPP`] dynamically. + /// + /// Returns a [`Token`] that ensures the OPP is automatically removed + /// when it goes out of scope. + #[inline] + pub fn add_opp(self, dev: &ARef) -> Result { + Token::new(dev, self) + } + + /// Returns the frequency associated with this OPP data. + #[inline] + fn freq(&self) -> Hertz { + Hertz(self.0.freq) + } +} + +/// [`OPP`] search options. +/// +/// # Examples +/// +/// Defines how to search for an [`OPP`] in a [`Table`] relative to a frequency. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{OPP, SearchType, Table}; +/// use kernel::sync::aref::ARef; +/// +/// fn find_opp(table: &Table, freq: Hertz) -> Result> { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// pr_info!("OPP frequency is: {:?}\n", opp.freq(None)); +/// pr_info!("OPP voltage is: {:?}\n", opp.voltage()); +/// pr_info!("OPP level is: {}\n", opp.level()); +/// pr_info!("OPP power is: {:?}\n", opp.power()); +/// +/// Ok(opp) +/// } +/// ``` +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum SearchType { + /// Match the exact frequency. + Exact, + /// Find the highest frequency less than or equal to the given value. + Floor, + /// Find the lowest frequency greater than or equal to the given value. + Ceil, +} + +/// OPP configuration callbacks. +/// +/// Implement this trait to customize OPP clock and regulator setup for your device. +#[vtable] +pub trait ConfigOps { + /// This is typically used to scale clocks when transitioning between OPPs. + #[inline] + fn config_clks(_dev: &Device, _table: &Table, _opp: &OPP, _scaling_down: bool) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// This provides access to the old and new OPPs, allowing for safe regulator adjustments. + #[inline] + fn config_regulators( + _dev: &Device, + _opp_old: &OPP, + _opp_new: &OPP, + _data: *mut *mut bindings::regulator, + _count: u32, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// OPP configuration token. +/// +/// Returned by the OPP core when configuration is applied to a [`Device`]. The associated +/// configuration is automatically cleared when the token is dropped. +pub struct ConfigToken(i32); + +impl Drop for ConfigToken { + fn drop(&mut self) { + // SAFETY: This is the same token value returned by the C code via `dev_pm_opp_set_config`. + unsafe { bindings::dev_pm_opp_clear_config(self.0) }; + } +} + +/// OPP configurations. +/// +/// Rust abstraction for the C `struct dev_pm_opp_config`. +/// +/// # Examples +/// +/// The following example demonstrates how to set OPP property-name configuration for a [`Device`]. +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Config, ConfigOps, ConfigToken}; +/// use kernel::str::CString; +/// use kernel::sync::aref::ARef; +/// use kernel::macros::vtable; +/// +/// #[derive(Default)] +/// struct Driver; +/// +/// #[vtable] +/// impl ConfigOps for Driver {} +/// +/// fn configure(dev: &ARef) -> Result { +/// let name = CString::try_from_fmt(fmt!("slow"))?; +/// +/// // The OPP configuration is cleared once the [`ConfigToken`] goes out of scope. +/// Config::::new() +/// .set_prop_name(name)? +/// .set(dev) +/// } +/// ``` +#[derive(Default)] +pub struct Config +where + T: Default, +{ + clk_names: Option>, + prop_name: Option, + regulator_names: Option>, + supported_hw: Option>, + + // Tuple containing (required device, index) + required_dev: Option<(ARef, u32)>, + _data: PhantomData, +} + +impl Config { + /// Creates a new instance of [`Config`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Initializes clock names. + pub fn set_clk_names(mut self, names: KVec) -> Result { + if self.clk_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.clk_names = Some(names); + Ok(self) + } + + /// Initializes property name. + pub fn set_prop_name(mut self, name: CString) -> Result { + if self.prop_name.is_some() { + return Err(EBUSY); + } + + self.prop_name = Some(name); + Ok(self) + } + + /// Initializes regulator names. + pub fn set_regulator_names(mut self, names: KVec) -> Result { + if self.regulator_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.regulator_names = Some(names); + + Ok(self) + } + + /// Initializes required devices. + pub fn set_required_dev(mut self, dev: ARef, index: u32) -> Result { + if self.required_dev.is_some() { + return Err(EBUSY); + } + + self.required_dev = Some((dev, index)); + Ok(self) + } + + /// Initializes supported hardware. + pub fn set_supported_hw(mut self, hw: KVec) -> Result { + if self.supported_hw.is_some() { + return Err(EBUSY); + } + + if hw.is_empty() { + return Err(EINVAL); + } + + self.supported_hw = Some(hw); + Ok(self) + } + + /// Sets the configuration with the OPP core. + /// + /// The returned [`ConfigToken`] will remove the configuration when dropped. + pub fn set(self, dev: &Device) -> Result { + let clk_names = self.clk_names.as_deref().map(to_c_str_array).transpose()?; + let regulator_names = self + .regulator_names + .as_deref() + .map(to_c_str_array) + .transpose()?; + + let set_config = || { + let clk_names = clk_names.as_ref().map_or(ptr::null(), |c| c.as_ptr()); + let regulator_names = regulator_names.as_ref().map_or(ptr::null(), |c| c.as_ptr()); + + let prop_name = self + .prop_name + .as_ref() + .map_or(ptr::null(), |p| p.as_char_ptr()); + + let (supported_hw, supported_hw_count) = self + .supported_hw + .as_ref() + .map_or((ptr::null(), 0), |hw| (hw.as_ptr(), hw.len() as u32)); + + let (required_dev, required_dev_index) = self + .required_dev + .as_ref() + .map_or((ptr::null_mut(), 0), |(dev, idx)| (dev.as_raw(), *idx)); + + let mut config = bindings::dev_pm_opp_config { + clk_names, + config_clks: if T::HAS_CONFIG_CLKS { + Some(Self::config_clks) + } else { + None + }, + prop_name, + regulator_names, + config_regulators: if T::HAS_CONFIG_REGULATORS { + Some(Self::config_regulators) + } else { + None + }, + supported_hw, + supported_hw_count, + + required_dev, + required_dev_index, + }; + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The OPP core guarantees not to access fields of [`Config`] after this + // call and so we don't need to save a copy of them for future use. + let ret = unsafe { bindings::dev_pm_opp_set_config(dev.as_raw(), &mut config) }; + + to_result(ret).map(|()| ConfigToken(ret)) + }; + + // Ensure the closure does not accidentally drop owned data; if violated, the compiler + // produces E0525 with e.g.: + // + // ``` + // closure is `FnOnce` because it moves the variable `clk_names` out of its environment + // ``` + let _: &dyn Fn() -> _ = &set_config; + + set_config() + } + + /// Config's clk callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_clks( + dev: *mut bindings::device, + opp_table: *mut bindings::opp_table, + opp: *mut bindings::dev_pm_opp, + _data: *mut c_void, + scaling_down: bool, + ) -> c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_clks( + &dev, + // SAFETY: 'opp_table' is guaranteed by the C code to be valid. + &unsafe { Table::from_raw_table(opp_table, &dev) }, + // SAFETY: 'opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(opp)? }, + scaling_down, + ) + .map(|()| 0) + }) + } + + /// Config's regulator callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_regulators( + dev: *mut bindings::device, + old_opp: *mut bindings::dev_pm_opp, + new_opp: *mut bindings::dev_pm_opp, + regulators: *mut *mut bindings::regulator, + count: c_uint, + ) -> c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_regulators( + &dev, + // SAFETY: 'old_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(old_opp)? }, + // SAFETY: 'new_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(new_opp)? }, + regulators, + count, + ) + .map(|()| 0) + }) + } +} + +/// A reference-counted OPP table. +/// +/// Rust abstraction for the C `struct opp_table`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`Table`]. +/// +/// Instances of this type are reference-counted. +/// +/// # Examples +/// +/// The following example demonstrates how to get OPP [`Table`] for a [`Cpumask`] and set its +/// frequency. +/// +/// ``` +/// # #![cfg(CONFIG_OF)] +/// use kernel::clk::Hertz; +/// use kernel::cpumask::Cpumask; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::Table; +/// use kernel::sync::aref::ARef; +/// +/// fn get_table(dev: &ARef, mask: &mut Cpumask, freq: Hertz) -> Result { +/// let mut opp_table = Table::from_of_cpumask(dev, mask)?; +/// +/// if opp_table.opp_count()? == 0 { +/// return Err(EINVAL); +/// } +/// +/// pr_info!("Max transition latency is: {} ns\n", opp_table.max_transition_latency_ns()); +/// pr_info!("Suspend frequency is: {:?}\n", opp_table.suspend_freq()); +/// +/// opp_table.set_rate(freq)?; +/// Ok(opp_table) +/// } +/// ``` +pub struct Table { + ptr: *mut bindings::opp_table, + dev: ARef, + #[allow(dead_code)] + em: bool, + #[allow(dead_code)] + of: bool, + cpus: Option, +} + +/// SAFETY: It is okay to send ownership of [`Table`] across thread boundaries. +unsafe impl Send for Table {} + +/// SAFETY: It is okay to access [`Table`] through shared references from other threads because +/// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for Table {} + +impl Table { + /// Creates a new reference-counted [`Table`] from a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid and non-null. + unsafe fn from_raw_table(ptr: *mut bindings::opp_table, dev: &ARef) -> Self { + // SAFETY: By the safety requirements, ptr is valid and its refcount will be incremented. + // + // INVARIANT: The reference-count is decremented when [`Table`] goes out of scope. + unsafe { bindings::dev_pm_opp_get_opp_table_ref(ptr) }; + + Self { + ptr, + dev: dev.clone(), + em: false, + of: false, + cpus: None, + } + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`]. + pub fn from_dev(dev: &Device) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + let ptr = from_err_ptr(unsafe { bindings::dev_pm_opp_get_opp_table(dev.as_raw()) })?; + + Ok(Self { + ptr, + dev: dev.into(), + em: false, + of: false, + cpus: None, + }) + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of(dev: &ARef, index: i32) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_add_table_indexed(dev.as_raw(), index) })?; + + // Get the newly created [`Table`]. + let mut table = Self::from_dev(dev)?; + table.of = true; + + Ok(table) + } + + /// Remove device tree based [`Table`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of(&self) { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. We took the reference from [`from_of`] earlier, it is safe to drop the + // same now. + unsafe { bindings::dev_pm_opp_of_remove_table(self.dev.as_raw()) }; + } + + /// Creates a new reference-counted [`Table`] instance for a [`Cpumask`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of_cpumask(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The cpumask is valid and the returned pointer will be owned by the [`Table`] + // instance. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_cpumask_add_table(cpumask.as_raw()) })?; + + // Fetch the newly created table. + let mut table = Self::from_dev(dev)?; + table.cpus = Some(CpumaskVar::try_clone(cpumask)?); + + Ok(table) + } + + /// Remove device tree based [`Table`] for a [`Cpumask`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of_cpumask(&self, cpumask: &Cpumask) { + // SAFETY: The cpumask is valid and we took the reference from [`from_of_cpumask`] earlier, + // it is safe to drop the same now. + unsafe { bindings::dev_pm_opp_of_cpumask_remove_table(cpumask.as_raw()) }; + } + + /// Returns the number of [`OPP`]s in the [`Table`]. + pub fn opp_count(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + let ret = unsafe { bindings::dev_pm_opp_get_opp_count(self.dev.as_raw()) }; + + to_result(ret).map(|()| ret as u32) + } + + /// Returns max clock latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_clock_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_clock_latency(self.dev.as_raw()) } + } + + /// Returns max volt latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_volt_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_volt_latency(self.dev.as_raw()) } + } + + /// Returns max transition latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_transition_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_transition_latency(self.dev.as_raw()) } + } + + /// Returns the suspend [`OPP`]'s frequency. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + Hertz(unsafe { bindings::dev_pm_opp_get_suspend_opp_freq(self.dev.as_raw()) }) + } + + /// Synchronizes regulators used by the [`Table`]. + #[inline] + pub fn sync_regulators(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_sync_regulators(self.dev.as_raw()) }) + } + + /// Gets sharing CPUs. + #[inline] + pub fn sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) }) + } + + /// Sets sharing CPUs. + pub fn set_sharing_cpus(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_set_sharing_cpus(self.dev.as_raw(), cpumask.as_raw()) + })?; + + if let Some(mask) = self.cpus.as_mut() { + // Update the cpumask as this will be used while removing the table. + cpumask.copy(mask); + } + + Ok(()) + } + + /// Gets sharing CPUs from device tree. + #[cfg(CONFIG_OF)] + #[inline] + pub fn of_sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) + }) + } + + /// Updates the voltage value for an [`OPP`]. + #[inline] + pub fn adjust_voltage( + &self, + freq: Hertz, + volt: MicroVolt, + volt_min: MicroVolt, + volt_max: MicroVolt, + ) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_adjust_voltage( + self.dev.as_raw(), + freq.into(), + volt.into(), + volt_min.into(), + volt_max.into(), + ) + }) + } + + /// Creates [`FreqTable`] from [`Table`]. + #[cfg(CONFIG_CPU_FREQ)] + #[inline] + pub fn cpufreq_table(&mut self) -> Result { + FreqTable::new(self) + } + + /// Configures device with [`OPP`] matching the frequency value. + #[inline] + pub fn set_rate(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_rate(self.dev.as_raw(), freq.into()) }) + } + + /// Configures device with [`OPP`]. + #[inline] + pub fn set_opp(&self, opp: &OPP) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_opp(self.dev.as_raw(), opp.as_raw()) }) + } + + /// Finds [`OPP`] based on frequency. + pub fn opp_from_freq( + &self, + freq: Hertz, + available: Option, + index: Option, + stype: SearchType, + ) -> Result> { + let raw_dev = self.dev.as_raw(); + let index = index.unwrap_or(0); + let mut rate = freq.into(); + + let ptr = from_err_ptr(match stype { + SearchType::Exact => { + if let Some(available) = available { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and + // its safety requirements. The returned pointer will be owned by the new + // [`OPP`] instance. + unsafe { + bindings::dev_pm_opp_find_freq_exact_indexed( + raw_dev, rate, index, available, + ) + } + } else { + return Err(EINVAL); + } + } + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_freq_ceil_indexed(raw_dev, &mut rate, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_freq_floor_indexed(raw_dev, &mut rate, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on level. + pub fn opp_from_level(&self, mut level: u32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Exact => unsafe { bindings::dev_pm_opp_find_level_exact(raw_dev, level) }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_level_ceil(raw_dev, &mut level) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_level_floor(raw_dev, &mut level) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on bandwidth. + pub fn opp_from_bw(&self, mut bw: u32, index: i32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // The OPP core doesn't support this yet. + SearchType::Exact => return Err(EINVAL), + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_bw_ceil(raw_dev, &mut bw, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_bw_floor(raw_dev, &mut bw, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Enables the [`OPP`]. + #[inline] + pub fn enable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_enable(self.dev.as_raw(), freq.into()) }) + } + + /// Disables the [`OPP`]. + #[inline] + pub fn disable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_disable(self.dev.as_raw(), freq.into()) }) + } + + /// Registers with the Energy model. + #[cfg(CONFIG_OF)] + pub fn of_register_em(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_register_em(self.dev.as_raw(), cpumask.as_raw()) + })?; + + self.em = true; + Ok(()) + } + + /// Unregisters with the Energy model. + #[cfg(all(CONFIG_OF, CONFIG_ENERGY_MODEL))] + #[inline] + fn of_unregister_em(&self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. We registered with the EM framework earlier, it is safe to unregister now. + unsafe { bindings::em_dev_unregister_perf_domain(self.dev.as_raw()) }; + } +} + +impl Drop for Table { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe + // to relinquish it now. + unsafe { bindings::dev_pm_opp_put_opp_table(self.ptr) }; + + #[cfg(CONFIG_OF)] + { + #[cfg(CONFIG_ENERGY_MODEL)] + if self.em { + self.of_unregister_em(); + } + + if self.of { + self.remove_of(); + } else if let Some(cpumask) = self.cpus.take() { + self.remove_of_cpumask(&cpumask); + } + } + } +} + +/// A reference-counted Operating performance point (OPP). +/// +/// Rust abstraction for the C `struct dev_pm_opp`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`OPP`]. +/// +/// Instances of this type are reference-counted. The reference count is incremented by the +/// `dev_pm_opp_get` function and decremented by `dev_pm_opp_put`. The Rust type `ARef` +/// represents a pointer that owns a reference count on the [`OPP`]. +/// +/// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code. +/// +/// # Examples +/// +/// The following example demonstrates how to get [`OPP`] corresponding to a frequency value and +/// configure the device with it. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{SearchType, Table}; +/// +/// fn configure_opp(table: &Table, freq: Hertz) -> Result { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// if opp.freq(None) != freq { +/// return Err(EINVAL); +/// } +/// +/// table.set_opp(&opp) +/// } +/// ``` +#[repr(transparent)] +pub struct OPP(Opaque); + +/// SAFETY: It is okay to send the ownership of [`OPP`] across thread boundaries. +unsafe impl Send for OPP {} + +/// SAFETY: It is okay to access [`OPP`] through shared references from other threads because we're +/// either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for OPP {} + +/// SAFETY: The type invariants guarantee that [`OPP`] is always refcounted. +unsafe impl AlwaysRefCounted for OPP { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_get(self.0.get()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_put(obj.cast().as_ptr()) } + } +} + +impl OPP { + /// Creates an owned reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is incremented by the C code and will be decremented by `dec_ref` when the + /// [`ARef`] object is dropped. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and the refcount of the [`OPP`] is incremented. + /// The caller must also ensure that it doesn't explicitly drop the refcount of the [`OPP`], as + /// the returned [`ARef`] object takes over the refcount increment on the underlying object and + /// the same will be dropped along with it. + pub unsafe fn from_raw_opp_owned(ptr: *mut bindings::dev_pm_opp) -> Result> { + let ptr = ptr::NonNull::new(ptr).ok_or(ENODEV)?; + + // SAFETY: The safety requirements guarantee the validity of the pointer. + // + // INVARIANT: The reference-count is decremented when [`OPP`] goes out of scope. + Ok(unsafe { ARef::from_raw(ptr.cast()) }) + } + + /// Creates a reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is not updated by the Rust API unless the returned reference is converted to + /// an [`ARef`] object. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the duration of `'a`. + #[inline] + pub unsafe fn from_raw_opp<'a>(ptr: *mut bindings::dev_pm_opp) -> Result<&'a Self> { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of 'a. The cast is okay because [`OPP`] is `repr(transparent)`. + Ok(unsafe { &*ptr.cast() }) + } + + #[inline] + fn as_raw(&self) -> *mut bindings::dev_pm_opp { + self.0.get() + } + + /// Returns the frequency of an [`OPP`]. + #[inline] + pub fn freq(&self, index: Option) -> Hertz { + let index = index.unwrap_or(0); + + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + Hertz(unsafe { bindings::dev_pm_opp_get_freq_indexed(self.as_raw(), index) }) + } + + /// Returns the voltage of an [`OPP`]. + #[inline] + pub fn voltage(&self) -> MicroVolt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroVolt(unsafe { bindings::dev_pm_opp_get_voltage(self.as_raw()) }) + } + + /// Returns the level of an [`OPP`]. + #[inline] + pub fn level(&self) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_level(self.as_raw()) } + } + + /// Returns the power of an [`OPP`]. + #[inline] + pub fn power(&self) -> MicroWatt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroWatt(unsafe { bindings::dev_pm_opp_get_power(self.as_raw()) }) + } + + /// Returns the required pstate of an [`OPP`]. + #[inline] + pub fn required_pstate(&self, index: u32) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_required_pstate(self.as_raw(), index) } + } + + /// Returns true if the [`OPP`] is turbo. + #[inline] + pub fn is_turbo(&self) -> bool { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_is_turbo(self.as_raw()) } + } +} diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs new file mode 100644 index 000000000..1c0796ea2 --- /dev/null +++ b/rust/kernel/page.rs @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel page allocation and management. + +use crate::{ + alloc::{ + AllocError, + Flags, // + }, + bindings, + error::{ + code::*, + Result, // + }, + uaccess::UserSliceReader, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::{ + self, + NonNull, // + }, // +}; + +/// A bitwise shift for the page size. +pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; + +/// The number of bytes in a page. +pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; + +/// A bitmask that gives the page containing a given address. +pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); + +/// Rounds up to the next multiple of [`PAGE_SIZE`]. +/// +/// Returns [`None`] on integer overflow. +/// +/// # Examples +/// +/// ``` +/// use kernel::page::{ +/// page_align, +/// PAGE_SIZE, +/// }; +/// +/// // Requested address is already aligned. +/// assert_eq!(page_align(0x0), Some(0x0)); +/// assert_eq!(page_align(PAGE_SIZE), Some(PAGE_SIZE)); +/// +/// // Requested address needs alignment up. +/// assert_eq!(page_align(0x1), Some(PAGE_SIZE)); +/// assert_eq!(page_align(PAGE_SIZE + 1), Some(2 * PAGE_SIZE)); +/// +/// // Requested address causes overflow (returns `None`). +/// let overflow_addr = usize::MAX - (PAGE_SIZE / 2); +/// assert_eq!(page_align(overflow_addr), None); +/// ``` +#[inline(always)] +pub const fn page_align(addr: usize) -> Option { + let Some(sum) = addr.checked_add(PAGE_SIZE - 1) else { + return None; + }; + Some(sum & PAGE_MASK) +} + +/// Representation of a non-owning reference to a [`Page`]. +/// +/// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a +/// [`Vmalloc`] allocation such as [`VBox`]. +/// +/// # Example +/// +/// ``` +/// # use kernel::{bindings, prelude::*}; +/// use kernel::page::{BorrowedPage, Page, PAGE_SIZE}; +/// # use core::{mem::MaybeUninit, ptr, ptr::NonNull }; +/// +/// fn borrow_page<'a>(vbox: &'a mut VBox>) -> BorrowedPage<'a> { +/// let ptr = ptr::from_ref(&**vbox); +/// +/// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. +/// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) }; +/// +/// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid +/// // pointer to `Vmalloc` memory. +/// let page = unsafe { NonNull::new_unchecked(page) }; +/// +/// // SAFETY: +/// // - `self.0` is a valid pointer to a `struct page`. +/// // - `self.0` is valid for the entire lifetime of `self`. +/// unsafe { BorrowedPage::from_raw(page) } +/// } +/// +/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; +/// let page = borrow_page(&mut vbox); +/// +/// // SAFETY: There is no concurrent read or write to this page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`. +/// +/// [`VBox`]: kernel::alloc::VBox +/// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc +pub struct BorrowedPage<'a>(ManuallyDrop, PhantomData<&'a Page>); + +impl<'a> BorrowedPage<'a> { + /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. + /// + /// # Safety + /// + /// - `ptr` must point to a valid `bindings::page`. + /// - `ptr` must remain valid for the entire lifetime `'a`. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + let page = Page { page: ptr }; + + // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime + // `'a`. + Self(ManuallyDrop::new(page), PhantomData) + } +} + +impl<'a> Deref for BorrowedPage<'a> { + type Target = Page; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Trait to be implemented by types which provide an [`Iterator`] implementation of +/// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). +pub trait AsPageIter { + /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. + fn page_iter(&mut self) -> Self::Iter<'_>; +} + +/// A pointer to a page that owns the page allocation. +/// +/// # Invariants +/// +/// The pointer is valid, and has ownership over the page. +pub struct Page { + page: NonNull, +} + +// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across +// threads is safe. +unsafe impl Send for Page {} + +// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing +// them concurrently is safe. +unsafe impl Sync for Page {} + +impl Page { + /// Allocates a new page. + /// + /// # Examples + /// + /// Allocate memory for a page. + /// + /// ``` + /// use kernel::page::Page; + /// + /// let page = Page::alloc_page(GFP_KERNEL)?; + /// # Ok::<(), kernel::alloc::AllocError>(()) + /// ``` + /// + /// Allocate memory for a page and zero its contents. + /// + /// ``` + /// use kernel::page::Page; + /// + /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + /// # Ok::<(), kernel::alloc::AllocError>(()) + /// ``` + #[inline] + pub fn alloc_page(flags: Flags) -> Result { + // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it + // is always safe to call this method. + let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; + let page = NonNull::new(page).ok_or(AllocError)?; + // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly + // allocated page. We transfer that ownership to the new `Page` object. + Ok(Self { page }) + } + + /// Returns a raw pointer to the page. + pub fn as_ptr(&self) -> *mut bindings::page { + self.page.as_ptr() + } + + /// Get the node id containing this page. + #[inline] + pub fn nid(&self) -> i32 { + // SAFETY: Always safe to call with a valid page. + unsafe { bindings::page_to_nid(self.as_ptr()) } + } + + /// Runs a piece of code with this page mapped to an address. + /// + /// The page is unmapped when this call returns. + /// + /// # Using the raw pointer + /// + /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for + /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might + /// only be mapped on the current thread, and when that is the case, dereferencing it on other + /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't + /// cause data races, the memory may be uninitialized, and so on. + /// + /// If multiple threads map the same page at the same time, then they may reference with + /// different addresses. However, even if the addresses are different, the underlying memory is + /// still the same for these purposes (e.g., it's still a data race if they both write to the + /// same underlying byte at the same time). + fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { + // SAFETY: `page` is valid due to the type invariants on `Page`. + let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; + + let res = f(mapped_addr.cast()); + + // This unmaps the page mapped above. + // + // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner + // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. + // + // In other words, if this call to `kunmap_local` happens when a different page should be + // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the + // call just above in `with_page_mapped` that made that possible. In this case, it is the + // unsafe block that wraps that other call that is incorrect. + unsafe { bindings::kunmap_local(mapped_addr) }; + + res + } + + /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. + /// + /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the + /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on + /// this task, as this method uses a local mapping. + /// + /// If `off` and `len` refers to a region outside of this page, then this method returns + /// [`EINVAL`] and does not call `f`. + /// + /// # Using the raw pointer + /// + /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for + /// `len` bytes and for the duration in which the closure is called. The pointer might only be + /// mapped on the current thread, and when that is the case, dereferencing it on other threads + /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause + /// data races, the memory may be uninitialized, and so on. + /// + /// If multiple threads map the same page at the same time, then they may reference with + /// different addresses. However, even if the addresses are different, the underlying memory is + /// still the same for these purposes (e.g., it's still a data race if they both write to the + /// same underlying byte at the same time). + fn with_pointer_into_page( + &self, + off: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, + ) -> Result { + let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; + + if bounds_ok { + self.with_page_mapped(move |page_addr| { + // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will + // result in a pointer that is in bounds or one off the end of the page. + f(unsafe { page_addr.add(off) }) + }) + } else { + Err(EINVAL) + } + } + + /// Maps the page and reads from it into the given buffer. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// * Callers must ensure that `dst` is valid for writing `len` bytes. + /// * Callers must ensure that this call does not race with a write to the same page that + /// overlaps with this read. + pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |src| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then + // it has performed a bounds check and guarantees that `src` is + // valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::copy_nonoverlapping(src, dst, len) }; + Ok(()) + }) + } + + /// Maps the page and writes into it from the given buffer. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// * Callers must ensure that `src` is valid for reading `len` bytes. + /// * Callers must ensure that this call does not race with a read or write to the same page + /// that overlaps with this write. + pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::copy_nonoverlapping(src, dst, len) }; + Ok(()) + }) + } + + /// Maps the page and zeroes the given slice. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// # Safety + /// + /// Callers must ensure that this call does not race with a read or write to the same page that + /// overlaps with this write. + pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. + // + // There caller guarantees that there is no data race. + unsafe { ptr::write_bytes(dst, 0u8, len) }; + Ok(()) + }) + } + + /// Copies data from userspace into this page. + /// + /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes + /// outside of the page, then this call returns [`EINVAL`]. + /// + /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. + /// However, they are not allowed on the page you are copying into. + /// + /// # Safety + /// + /// Callers must ensure that this call does not race with a read or write to the same page that + /// overlaps with this write. + pub unsafe fn copy_from_user_slice_raw( + &self, + reader: &mut UserSliceReader, + offset: usize, + len: usize, + ) -> Result { + self.with_pointer_into_page(offset, len, move |dst| { + // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a + // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have + // exclusive access to the slice since the caller guarantees that there are no races. + reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) + }) + } +} + +impl Drop for Page { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, we have ownership of the page and can free it. + unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; + } +} diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs new file mode 100644 index 000000000..3ec897709 --- /dev/null +++ b/rust/kernel/pci.rs @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the PCI bus. +//! +//! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) + +use crate::{ + bindings, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + str::CStr, + types::Opaque, + ThisModule, // +}; +use core::{ + marker::PhantomData, + mem::offset_of, + num::NonZero, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +mod id; +mod io; +mod irq; + +pub use self::id::{ + Class, + ClassMask, + Vendor, // +}; +pub use self::io::{ + Bar, + ConfigSpace, + ConfigSpaceSize, + DevresBar, + Extended, + Normal, // +}; +pub use self::irq::{ + IrqType, + IrqTypes, + IrqVector, + IrqVectorRegistration, // +}; + +/// An adapter for the registration of PCI drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::pci_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct pci_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::pci_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + pdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct pci_driver` on initialization. + unsafe { + (*pdrv.get()).name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__pci_register_driver(pdrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(pdrv: &Opaque) { + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::pci_unregister_driver(pdrv.get()) } + } +} + +impl Adapter { + extern "C" fn probe_callback( + pdev: *mut bindings::pci_dev, + id: *const bindings::pci_device_id, + ) -> c_int { + // SAFETY: The PCI bus only ever calls the probe callback with a valid pointer to a + // `struct pci_dev`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>` or + // `pci_device_id_any` which has 0 as driver_data. It can also come from dynamic IDs, which + // will ensure that `driver_data` exists in `T::ID_TABLE`. + let info = unsafe { id.info_unchecked_opt::() }; + + from_result(|| { + let data = T::probe(pdev, info); + + pdev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { + // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a + // `struct pci_dev`. + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; + + T::unbind(pdev, data); + } +} + +/// Declares a kernel module that exposes a single PCI driver. +/// +/// # Examples +/// +///```ignore +/// kernel::module_pci_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +///``` +#[macro_export] +macro_rules! module_pci_driver { +($($f:tt)*) => { + $crate::module_driver!(, $crate::pci::Adapter, { $($f)* }); +}; +} + +/// Abstraction for the PCI device ID structure ([`struct pci_device_id`]). +/// +/// [`struct pci_device_id`]: https://docs.kernel.org/PCI/pci.html#c.pci_device_id +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::pci_device_id); + +impl DeviceId { + const PCI_ANY_ID: u32 = !0; + + /// Equivalent to C's `PCI_DEVICE` macro. + /// + /// Create a new `pci::DeviceId` from a vendor and device ID. + #[inline] + pub const fn from_id(vendor: Vendor, device: u32) -> Self { + Self(bindings::pci_device_id { + vendor: vendor.as_raw() as u32, + device, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class: 0, + class_mask: 0, + driver_data: 0, + override_only: 0, + }) + } + + /// Equivalent to C's `PCI_DEVICE_CLASS` macro. + /// + /// Create a new `pci::DeviceId` from a class number and mask. + #[inline] + pub const fn from_class(class: u32, class_mask: u32) -> Self { + Self(bindings::pci_device_id { + vendor: DeviceId::PCI_ANY_ID, + device: DeviceId::PCI_ANY_ID, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class, + class_mask, + driver_data: 0, + override_only: 0, + }) + } + + /// Create a new [`DeviceId`] from a class number, mask, and specific vendor. + /// + /// This is more targeted than [`DeviceId::from_class`]: in addition to matching by [`Vendor`], + /// it also matches the PCI [`Class`] (up to the entire 24 bits, depending on the + /// [`ClassMask`]). + #[inline] + pub const fn from_class_and_vendor( + class: Class, + class_mask: ClassMask, + vendor: Vendor, + ) -> Self { + Self(bindings::pci_device_id { + vendor: vendor.as_raw() as u32, + device: DeviceId::PCI_ANY_ID, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class: class.as_raw(), + class_mask: class_mask.as_raw(), + driver_data: 0, + override_only: 0, + }) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `pci_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::pci_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); +} + +/// `IdTable` type for PCI. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a PCI `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! pci_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("pci", $crate::pci::DeviceId, $($tt)*); + }; +} + +/// The PCI driver trait. +/// +/// # Examples +/// +///``` +/// # use kernel::{bindings, device::Core, pci}; +/// +/// struct MyDriver; +/// +/// kernel::pci_device_table!( +/// PCI_TABLE, +/// ::IdInfo, +/// [ +/// ( +/// pci::DeviceId::from_id(pci::Vendor::REDHAT, bindings::PCI_ANY_ID as u32), +/// (), +/// ) +/// ] +/// ); +/// +/// impl pci::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const ID_TABLE: pci::IdTable = &PCI_TABLE; +/// +/// fn probe<'bound>( +/// _pdev: &'bound pci::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// } +///``` +/// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the +/// `Adapter` documentation for an example. +pub trait Driver { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// PCI driver probe. + /// + /// Called when a new pci device is added or discovered. Implementers should + /// attempt to initialize the device here. + fn probe<'bound>( + dev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// PCI driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The PCI device representation. +/// +/// This structure represents the Rust abstraction for a C `struct pci_dev`. The implementation +/// abstracts the usage of an already existing C `struct pci_dev` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct pci_dev` created by the C portion of the +/// kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::pci_dev { + self.0.get() + } +} + +impl Device { + /// Returns the PCI vendor ID as [`Vendor`]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{device::Core, pci::{self, Vendor}, prelude::*}; + /// fn log_device_info(pdev: &pci::Device>) -> Result { + /// // Get an instance of `Vendor`. + /// let vendor = pdev.vendor_id(); + /// dev_info!( + /// pdev, + /// "Device: Vendor={}, Device=0x{:x}\n", + /// vendor, + /// pdev.device_id() + /// ); + /// Ok(()) + /// } + /// ``` + #[inline] + pub fn vendor_id(&self) -> Vendor { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + let vendor_id = unsafe { (*self.as_raw()).vendor }; + Vendor::from_raw(vendor_id) + } + + /// Returns the PCI device ID. + #[inline] + pub fn device_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).device } + } + + /// Returns the PCI revision ID. + #[inline] + pub fn revision_id(&self) -> u8 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).revision } + } + + /// Returns the PCI bus device/function. + #[inline] + pub fn dev_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { bindings::pci_dev_id(self.as_raw()) } + } + + /// Returns the PCI subsystem vendor ID. + #[inline] + pub fn subsystem_vendor_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).subsystem_vendor } + } + + /// Returns the PCI subsystem device ID. + #[inline] + pub fn subsystem_device_id(&self) -> u16 { + // SAFETY: By its type invariant `self.as_raw` is always a valid pointer to a + // `struct pci_dev`. + unsafe { (*self.as_raw()).subsystem_device } + } + + /// Returns the start of the given PCI BAR resource. + pub fn resource_start(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + Ok(unsafe { bindings::pci_resource_start(self.as_raw(), bar.try_into()?) }) + } + + /// Returns the size of the given PCI BAR resource. + pub fn resource_len(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) + } + + /// Returns the PCI class as a `Class` struct. + #[inline] + pub fn pci_class(&self) -> Class { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + Class::from_raw(unsafe { (*self.as_raw()).class }) + } +} + +impl<'a> Device> { + /// Returns the total number of VFs, or [`None`] if SR-IOV is not available. + #[inline] + pub fn sriov_get_totalvfs(&self) -> Option> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct pci_dev`. + let total_vfs = unsafe { bindings::pci_sriov_get_totalvfs(self.as_raw()) }; + + // CAST: The C function returns `unsigned int`, but the value originates + // from TotalVFs/driver_max_VFs (which are defined as `u16`), so this cast + // cannot truncate. + NonZero::new(total_vfs as u16) + } + + /// Enable memory resources for this device. + pub fn enable_device_mem(&self) -> Result { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + to_result(unsafe { bindings::pci_enable_device_mem(self.as_raw()) }) + } + + /// Enable bus-mastering for this device. + #[inline] + pub fn set_master(&self) { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + unsafe { bindings::pci_set_master(self.as_raw()) }; + } +} + +// SAFETY: `pci::Device` is a transparent wrapper of `struct pci_dev`. +// The offset is guaranteed to point to a valid device field inside `pci::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::pci_dev, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +impl<'a> crate::dma::Device<'a> for Device> {} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::pci_dev_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct pci_dev`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &Device { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if !unsafe { bindings::dev_is_pci(dev.as_raw()) } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the bus type of `dev` equals `bindings::pci_bus_type`, + // hence `dev` must be embedded in a valid `struct pci_dev` as guaranteed by the + // corresponding C code. + let pdev = unsafe { container_of!(dev.as_raw(), bindings::pci_dev, dev) }; + + // SAFETY: `pdev` is a valid pointer to a `struct pci_dev`. + Ok(unsafe { &*pdev.cast() }) + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct pci_dev` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs new file mode 100644 index 000000000..dbaf30166 --- /dev/null +++ b/rust/kernel/pci/id.rs @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI device identifiers and related types. +//! +//! This module contains PCI class codes, Vendor IDs, and supporting types. + +use crate::{ + bindings, + fmt, + prelude::*, // +}; + +/// PCI device class codes. +/// +/// Each entry contains the full 24-bit PCI class code (base class in bits +/// 23-16, subclass in bits 15-8, programming interface in bits 7-0). +/// +/// # Examples +/// +/// ``` +/// # use kernel::{device::Core, pci::{self, Class}, prelude::*}; +/// fn probe_device(pdev: &pci::Device>) -> Result { +/// let pci_class = pdev.pci_class(); +/// dev_info!( +/// pdev, +/// "Detected PCI class: {}\n", +/// pci_class +/// ); +/// Ok(()) +/// } +/// ``` +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct Class(u32); + +/// PCI class mask constants for matching [`Class`] codes. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClassMask { + /// Match the full 24-bit class code. + Full = 0xffffff, + /// Match the upper 16 bits of the class code (base class and subclass only) + ClassSubclass = 0xffff00, +} + +macro_rules! define_all_pci_classes { + ( + $($variant:ident = $binding:expr,)+ + ) => { + impl Class { + $( + #[allow(missing_docs)] + pub const $variant: Self = Self(Self::to_24bit_class($binding)); + )+ + } + + impl fmt::Display for Class { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + $( + &Self::$variant => write!(f, stringify!($variant)), + )+ + _ => ::fmt(self, f), + } + } + } + }; +} + +/// Once constructed, a [`Class`] contains a valid PCI class code. +impl Class { + /// Create a [`Class`] from a raw 24-bit class code. + #[inline] + pub(super) fn from_raw(class_code: u32) -> Self { + Self(class_code) + } + + /// Get the raw 24-bit class code value. + #[inline] + pub const fn as_raw(self) -> u32 { + self.0 + } + + // Converts a PCI class constant to 24-bit format. + // + // Many device drivers use only the upper 16 bits (base class and subclass), + // but some use the full 24 bits. In order to support both cases, store the + // class code as a 24-bit value, where 16-bit values are shifted up 8 bits. + const fn to_24bit_class(val: u32) -> u32 { + if val > 0xFFFF { + val + } else { + val << 8 + } + } +} + +impl fmt::Debug for Class { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "0x{:06x}", self.0) + } +} + +impl ClassMask { + /// Get the raw mask value. + #[inline] + pub const fn as_raw(self) -> u32 { + self as u32 + } +} + +impl TryFrom for ClassMask { + type Error = Error; + + fn try_from(value: u32) -> Result { + match value { + 0xffffff => Ok(ClassMask::Full), + 0xffff00 => Ok(ClassMask::ClassSubclass), + _ => Err(EINVAL), + } + } +} + +/// PCI vendor IDs. +/// +/// Each entry contains the 16-bit PCI vendor ID as assigned by the PCI SIG. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct Vendor(u16); + +macro_rules! define_all_pci_vendors { + ( + $($variant:ident = $binding:expr,)+ + ) => { + impl Vendor { + $( + #[allow(missing_docs)] + pub const $variant: Self = Self($binding as u16); + )+ + } + + impl fmt::Display for Vendor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + $( + &Self::$variant => write!(f, stringify!($variant)), + )+ + _ => ::fmt(self, f), + } + } + } + }; +} + +/// Once constructed, a `Vendor` contains a valid PCI Vendor ID. +impl Vendor { + /// Create a Vendor from a raw 16-bit vendor ID. + #[inline] + pub(super) fn from_raw(vendor_id: u16) -> Self { + Self(vendor_id) + } + + /// Get the raw 16-bit vendor ID value. + #[inline] + pub const fn as_raw(self) -> u16 { + self.0 + } +} + +impl fmt::Debug for Vendor { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "0x{:04x}", self.0) + } +} + +define_all_pci_classes! { + NOT_DEFINED = bindings::PCI_CLASS_NOT_DEFINED, // 0x000000 + NOT_DEFINED_VGA = bindings::PCI_CLASS_NOT_DEFINED_VGA, // 0x000100 + + STORAGE_SCSI = bindings::PCI_CLASS_STORAGE_SCSI, // 0x010000 + STORAGE_IDE = bindings::PCI_CLASS_STORAGE_IDE, // 0x010100 + STORAGE_FLOPPY = bindings::PCI_CLASS_STORAGE_FLOPPY, // 0x010200 + STORAGE_IPI = bindings::PCI_CLASS_STORAGE_IPI, // 0x010300 + STORAGE_RAID = bindings::PCI_CLASS_STORAGE_RAID, // 0x010400 + STORAGE_SATA = bindings::PCI_CLASS_STORAGE_SATA, // 0x010600 + STORAGE_SATA_AHCI = bindings::PCI_CLASS_STORAGE_SATA_AHCI, // 0x010601 + STORAGE_SAS = bindings::PCI_CLASS_STORAGE_SAS, // 0x010700 + STORAGE_EXPRESS = bindings::PCI_CLASS_STORAGE_EXPRESS, // 0x010802 + STORAGE_OTHER = bindings::PCI_CLASS_STORAGE_OTHER, // 0x018000 + + NETWORK_ETHERNET = bindings::PCI_CLASS_NETWORK_ETHERNET, // 0x020000 + NETWORK_TOKEN_RING = bindings::PCI_CLASS_NETWORK_TOKEN_RING, // 0x020100 + NETWORK_FDDI = bindings::PCI_CLASS_NETWORK_FDDI, // 0x020200 + NETWORK_ATM = bindings::PCI_CLASS_NETWORK_ATM, // 0x020300 + NETWORK_OTHER = bindings::PCI_CLASS_NETWORK_OTHER, // 0x028000 + + DISPLAY_VGA = bindings::PCI_CLASS_DISPLAY_VGA, // 0x030000 + DISPLAY_XGA = bindings::PCI_CLASS_DISPLAY_XGA, // 0x030100 + DISPLAY_3D = bindings::PCI_CLASS_DISPLAY_3D, // 0x030200 + DISPLAY_OTHER = bindings::PCI_CLASS_DISPLAY_OTHER, // 0x038000 + + MULTIMEDIA_VIDEO = bindings::PCI_CLASS_MULTIMEDIA_VIDEO, // 0x040000 + MULTIMEDIA_AUDIO = bindings::PCI_CLASS_MULTIMEDIA_AUDIO, // 0x040100 + MULTIMEDIA_PHONE = bindings::PCI_CLASS_MULTIMEDIA_PHONE, // 0x040200 + MULTIMEDIA_HD_AUDIO = bindings::PCI_CLASS_MULTIMEDIA_HD_AUDIO, // 0x040300 + MULTIMEDIA_OTHER = bindings::PCI_CLASS_MULTIMEDIA_OTHER, // 0x048000 + + MEMORY_RAM = bindings::PCI_CLASS_MEMORY_RAM, // 0x050000 + MEMORY_FLASH = bindings::PCI_CLASS_MEMORY_FLASH, // 0x050100 + MEMORY_CXL = bindings::PCI_CLASS_MEMORY_CXL, // 0x050200 + MEMORY_OTHER = bindings::PCI_CLASS_MEMORY_OTHER, // 0x058000 + + BRIDGE_HOST = bindings::PCI_CLASS_BRIDGE_HOST, // 0x060000 + BRIDGE_ISA = bindings::PCI_CLASS_BRIDGE_ISA, // 0x060100 + BRIDGE_EISA = bindings::PCI_CLASS_BRIDGE_EISA, // 0x060200 + BRIDGE_MC = bindings::PCI_CLASS_BRIDGE_MC, // 0x060300 + BRIDGE_PCI_NORMAL = bindings::PCI_CLASS_BRIDGE_PCI_NORMAL, // 0x060400 + BRIDGE_PCI_SUBTRACTIVE = bindings::PCI_CLASS_BRIDGE_PCI_SUBTRACTIVE, // 0x060401 + BRIDGE_PCMCIA = bindings::PCI_CLASS_BRIDGE_PCMCIA, // 0x060500 + BRIDGE_NUBUS = bindings::PCI_CLASS_BRIDGE_NUBUS, // 0x060600 + BRIDGE_CARDBUS = bindings::PCI_CLASS_BRIDGE_CARDBUS, // 0x060700 + BRIDGE_RACEWAY = bindings::PCI_CLASS_BRIDGE_RACEWAY, // 0x060800 + BRIDGE_OTHER = bindings::PCI_CLASS_BRIDGE_OTHER, // 0x068000 + + COMMUNICATION_SERIAL = bindings::PCI_CLASS_COMMUNICATION_SERIAL, // 0x070000 + COMMUNICATION_PARALLEL = bindings::PCI_CLASS_COMMUNICATION_PARALLEL, // 0x070100 + COMMUNICATION_MULTISERIAL = bindings::PCI_CLASS_COMMUNICATION_MULTISERIAL, // 0x070200 + COMMUNICATION_MODEM = bindings::PCI_CLASS_COMMUNICATION_MODEM, // 0x070300 + COMMUNICATION_OTHER = bindings::PCI_CLASS_COMMUNICATION_OTHER, // 0x078000 + + SYSTEM_PIC = bindings::PCI_CLASS_SYSTEM_PIC, // 0x080000 + SYSTEM_PIC_IOAPIC = bindings::PCI_CLASS_SYSTEM_PIC_IOAPIC, // 0x080010 + SYSTEM_PIC_IOXAPIC = bindings::PCI_CLASS_SYSTEM_PIC_IOXAPIC, // 0x080020 + SYSTEM_DMA = bindings::PCI_CLASS_SYSTEM_DMA, // 0x080100 + SYSTEM_TIMER = bindings::PCI_CLASS_SYSTEM_TIMER, // 0x080200 + SYSTEM_RTC = bindings::PCI_CLASS_SYSTEM_RTC, // 0x080300 + SYSTEM_PCI_HOTPLUG = bindings::PCI_CLASS_SYSTEM_PCI_HOTPLUG, // 0x080400 + SYSTEM_SDHCI = bindings::PCI_CLASS_SYSTEM_SDHCI, // 0x080500 + SYSTEM_RCEC = bindings::PCI_CLASS_SYSTEM_RCEC, // 0x080700 + SYSTEM_OTHER = bindings::PCI_CLASS_SYSTEM_OTHER, // 0x088000 + + INPUT_KEYBOARD = bindings::PCI_CLASS_INPUT_KEYBOARD, // 0x090000 + INPUT_PEN = bindings::PCI_CLASS_INPUT_PEN, // 0x090100 + INPUT_MOUSE = bindings::PCI_CLASS_INPUT_MOUSE, // 0x090200 + INPUT_SCANNER = bindings::PCI_CLASS_INPUT_SCANNER, // 0x090300 + INPUT_GAMEPORT = bindings::PCI_CLASS_INPUT_GAMEPORT, // 0x090400 + INPUT_OTHER = bindings::PCI_CLASS_INPUT_OTHER, // 0x098000 + + DOCKING_GENERIC = bindings::PCI_CLASS_DOCKING_GENERIC, // 0x0a0000 + DOCKING_OTHER = bindings::PCI_CLASS_DOCKING_OTHER, // 0x0a8000 + + PROCESSOR_386 = bindings::PCI_CLASS_PROCESSOR_386, // 0x0b0000 + PROCESSOR_486 = bindings::PCI_CLASS_PROCESSOR_486, // 0x0b0100 + PROCESSOR_PENTIUM = bindings::PCI_CLASS_PROCESSOR_PENTIUM, // 0x0b0200 + PROCESSOR_ALPHA = bindings::PCI_CLASS_PROCESSOR_ALPHA, // 0x0b1000 + PROCESSOR_POWERPC = bindings::PCI_CLASS_PROCESSOR_POWERPC, // 0x0b2000 + PROCESSOR_MIPS = bindings::PCI_CLASS_PROCESSOR_MIPS, // 0x0b3000 + PROCESSOR_CO = bindings::PCI_CLASS_PROCESSOR_CO, // 0x0b4000 + + SERIAL_FIREWIRE = bindings::PCI_CLASS_SERIAL_FIREWIRE, // 0x0c0000 + SERIAL_FIREWIRE_OHCI = bindings::PCI_CLASS_SERIAL_FIREWIRE_OHCI, // 0x0c0010 + SERIAL_ACCESS = bindings::PCI_CLASS_SERIAL_ACCESS, // 0x0c0100 + SERIAL_SSA = bindings::PCI_CLASS_SERIAL_SSA, // 0x0c0200 + SERIAL_USB_UHCI = bindings::PCI_CLASS_SERIAL_USB_UHCI, // 0x0c0300 + SERIAL_USB_OHCI = bindings::PCI_CLASS_SERIAL_USB_OHCI, // 0x0c0310 + SERIAL_USB_EHCI = bindings::PCI_CLASS_SERIAL_USB_EHCI, // 0x0c0320 + SERIAL_USB_XHCI = bindings::PCI_CLASS_SERIAL_USB_XHCI, // 0x0c0330 + SERIAL_USB_CDNS = bindings::PCI_CLASS_SERIAL_USB_CDNS, // 0x0c0380 + SERIAL_USB_DEVICE = bindings::PCI_CLASS_SERIAL_USB_DEVICE, // 0x0c03fe + SERIAL_FIBER = bindings::PCI_CLASS_SERIAL_FIBER, // 0x0c0400 + SERIAL_SMBUS = bindings::PCI_CLASS_SERIAL_SMBUS, // 0x0c0500 + SERIAL_IPMI_SMIC = bindings::PCI_CLASS_SERIAL_IPMI_SMIC, // 0x0c0700 + SERIAL_IPMI_KCS = bindings::PCI_CLASS_SERIAL_IPMI_KCS, // 0x0c0701 + SERIAL_IPMI_BT = bindings::PCI_CLASS_SERIAL_IPMI_BT, // 0x0c0702 + + WIRELESS_RF_CONTROLLER = bindings::PCI_CLASS_WIRELESS_RF_CONTROLLER, // 0x0d1000 + WIRELESS_WHCI = bindings::PCI_CLASS_WIRELESS_WHCI, // 0x0d1010 + + INTELLIGENT_I2O = bindings::PCI_CLASS_INTELLIGENT_I2O, // 0x0e0000 + + SATELLITE_TV = bindings::PCI_CLASS_SATELLITE_TV, // 0x0f0000 + SATELLITE_AUDIO = bindings::PCI_CLASS_SATELLITE_AUDIO, // 0x0f0100 + SATELLITE_VOICE = bindings::PCI_CLASS_SATELLITE_VOICE, // 0x0f0300 + SATELLITE_DATA = bindings::PCI_CLASS_SATELLITE_DATA, // 0x0f0400 + + CRYPT_NETWORK = bindings::PCI_CLASS_CRYPT_NETWORK, // 0x100000 + CRYPT_ENTERTAINMENT = bindings::PCI_CLASS_CRYPT_ENTERTAINMENT, // 0x100100 + CRYPT_OTHER = bindings::PCI_CLASS_CRYPT_OTHER, // 0x108000 + + SP_DPIO = bindings::PCI_CLASS_SP_DPIO, // 0x110000 + SP_OTHER = bindings::PCI_CLASS_SP_OTHER, // 0x118000 + + ACCELERATOR_PROCESSING = bindings::PCI_CLASS_ACCELERATOR_PROCESSING, // 0x120000 + + OTHERS = bindings::PCI_CLASS_OTHERS, // 0xff0000 +} + +define_all_pci_vendors! { + PCI_SIG = bindings::PCI_VENDOR_ID_PCI_SIG, // 0x0001 + LOONGSON = bindings::PCI_VENDOR_ID_LOONGSON, // 0x0014 + SOLIDIGM = bindings::PCI_VENDOR_ID_SOLIDIGM, // 0x025e + TTTECH = bindings::PCI_VENDOR_ID_TTTECH, // 0x0357 + DYNALINK = bindings::PCI_VENDOR_ID_DYNALINK, // 0x0675 + UBIQUITI = bindings::PCI_VENDOR_ID_UBIQUITI, // 0x0777 + BERKOM = bindings::PCI_VENDOR_ID_BERKOM, // 0x0871 + ITTIM = bindings::PCI_VENDOR_ID_ITTIM, // 0x0b48 + COMPAQ = bindings::PCI_VENDOR_ID_COMPAQ, // 0x0e11 + LSI_LOGIC = bindings::PCI_VENDOR_ID_LSI_LOGIC, // 0x1000 + ATI = bindings::PCI_VENDOR_ID_ATI, // 0x1002 + VLSI = bindings::PCI_VENDOR_ID_VLSI, // 0x1004 + ADL = bindings::PCI_VENDOR_ID_ADL, // 0x1005 + NS = bindings::PCI_VENDOR_ID_NS, // 0x100b + TSENG = bindings::PCI_VENDOR_ID_TSENG, // 0x100c + WEITEK = bindings::PCI_VENDOR_ID_WEITEK, // 0x100e + DEC = bindings::PCI_VENDOR_ID_DEC, // 0x1011 + CIRRUS = bindings::PCI_VENDOR_ID_CIRRUS, // 0x1013 + IBM = bindings::PCI_VENDOR_ID_IBM, // 0x1014 + UNISYS = bindings::PCI_VENDOR_ID_UNISYS, // 0x1018 + COMPEX2 = bindings::PCI_VENDOR_ID_COMPEX2, // 0x101a + WD = bindings::PCI_VENDOR_ID_WD, // 0x101c + AMI = bindings::PCI_VENDOR_ID_AMI, // 0x101e + AMD = bindings::PCI_VENDOR_ID_AMD, // 0x1022 + TRIDENT = bindings::PCI_VENDOR_ID_TRIDENT, // 0x1023 + AI = bindings::PCI_VENDOR_ID_AI, // 0x1025 + DELL = bindings::PCI_VENDOR_ID_DELL, // 0x1028 + MATROX = bindings::PCI_VENDOR_ID_MATROX, // 0x102B + MOBILITY_ELECTRONICS = bindings::PCI_VENDOR_ID_MOBILITY_ELECTRONICS, // 0x14f2 + CT = bindings::PCI_VENDOR_ID_CT, // 0x102c + MIRO = bindings::PCI_VENDOR_ID_MIRO, // 0x1031 + NEC = bindings::PCI_VENDOR_ID_NEC, // 0x1033 + FD = bindings::PCI_VENDOR_ID_FD, // 0x1036 + SI = bindings::PCI_VENDOR_ID_SI, // 0x1039 + HP = bindings::PCI_VENDOR_ID_HP, // 0x103c + HP_3PAR = bindings::PCI_VENDOR_ID_HP_3PAR, // 0x1590 + PCTECH = bindings::PCI_VENDOR_ID_PCTECH, // 0x1042 + ASUSTEK = bindings::PCI_VENDOR_ID_ASUSTEK, // 0x1043 + DPT = bindings::PCI_VENDOR_ID_DPT, // 0x1044 + OPTI = bindings::PCI_VENDOR_ID_OPTI, // 0x1045 + ELSA = bindings::PCI_VENDOR_ID_ELSA, // 0x1048 + STMICRO = bindings::PCI_VENDOR_ID_STMICRO, // 0x104A + BUSLOGIC = bindings::PCI_VENDOR_ID_BUSLOGIC, // 0x104B + TI = bindings::PCI_VENDOR_ID_TI, // 0x104c + SONY = bindings::PCI_VENDOR_ID_SONY, // 0x104d + WINBOND2 = bindings::PCI_VENDOR_ID_WINBOND2, // 0x1050 + ANIGMA = bindings::PCI_VENDOR_ID_ANIGMA, // 0x1051 + EFAR = bindings::PCI_VENDOR_ID_EFAR, // 0x1055 + MOTOROLA = bindings::PCI_VENDOR_ID_MOTOROLA, // 0x1057 + PROMISE = bindings::PCI_VENDOR_ID_PROMISE, // 0x105a + FOXCONN = bindings::PCI_VENDOR_ID_FOXCONN, // 0x105b + UMC = bindings::PCI_VENDOR_ID_UMC, // 0x1060 + PICOPOWER = bindings::PCI_VENDOR_ID_PICOPOWER, // 0x1066 + MYLEX = bindings::PCI_VENDOR_ID_MYLEX, // 0x1069 + APPLE = bindings::PCI_VENDOR_ID_APPLE, // 0x106b + YAMAHA = bindings::PCI_VENDOR_ID_YAMAHA, // 0x1073 + QLOGIC = bindings::PCI_VENDOR_ID_QLOGIC, // 0x1077 + CYRIX = bindings::PCI_VENDOR_ID_CYRIX, // 0x1078 + CONTAQ = bindings::PCI_VENDOR_ID_CONTAQ, // 0x1080 + OLICOM = bindings::PCI_VENDOR_ID_OLICOM, // 0x108d + SUN = bindings::PCI_VENDOR_ID_SUN, // 0x108e + NI = bindings::PCI_VENDOR_ID_NI, // 0x1093 + CMD = bindings::PCI_VENDOR_ID_CMD, // 0x1095 + BROOKTREE = bindings::PCI_VENDOR_ID_BROOKTREE, // 0x109e + SGI = bindings::PCI_VENDOR_ID_SGI, // 0x10a9 + WINBOND = bindings::PCI_VENDOR_ID_WINBOND, // 0x10ad + PLX = bindings::PCI_VENDOR_ID_PLX, // 0x10b5 + MADGE = bindings::PCI_VENDOR_ID_MADGE, // 0x10b6 + THREECOM = bindings::PCI_VENDOR_ID_3COM, // 0x10b7 + AL = bindings::PCI_VENDOR_ID_AL, // 0x10b9 + NEOMAGIC = bindings::PCI_VENDOR_ID_NEOMAGIC, // 0x10c8 + TCONRAD = bindings::PCI_VENDOR_ID_TCONRAD, // 0x10da + ROHM = bindings::PCI_VENDOR_ID_ROHM, // 0x10db + NVIDIA = bindings::PCI_VENDOR_ID_NVIDIA, // 0x10de + IMS = bindings::PCI_VENDOR_ID_IMS, // 0x10e0 + AMCC = bindings::PCI_VENDOR_ID_AMCC, // 0x10e8 + AMPERE = bindings::PCI_VENDOR_ID_AMPERE, // 0x1def + INTERG = bindings::PCI_VENDOR_ID_INTERG, // 0x10ea + REALTEK = bindings::PCI_VENDOR_ID_REALTEK, // 0x10ec + XILINX = bindings::PCI_VENDOR_ID_XILINX, // 0x10ee + INIT = bindings::PCI_VENDOR_ID_INIT, // 0x1101 + CREATIVE = bindings::PCI_VENDOR_ID_CREATIVE, // 0x1102 + TTI = bindings::PCI_VENDOR_ID_TTI, // 0x1103 + SIGMA = bindings::PCI_VENDOR_ID_SIGMA, // 0x1105 + VIA = bindings::PCI_VENDOR_ID_VIA, // 0x1106 + SIEMENS = bindings::PCI_VENDOR_ID_SIEMENS, // 0x110A + VORTEX = bindings::PCI_VENDOR_ID_VORTEX, // 0x1119 + EF = bindings::PCI_VENDOR_ID_EF, // 0x111a + IDT = bindings::PCI_VENDOR_ID_IDT, // 0x111d + FORE = bindings::PCI_VENDOR_ID_FORE, // 0x1127 + PHILIPS = bindings::PCI_VENDOR_ID_PHILIPS, // 0x1131 + EICON = bindings::PCI_VENDOR_ID_EICON, // 0x1133 + CISCO = bindings::PCI_VENDOR_ID_CISCO, // 0x1137 + ZIATECH = bindings::PCI_VENDOR_ID_ZIATECH, // 0x1138 + SYSKONNECT = bindings::PCI_VENDOR_ID_SYSKONNECT, // 0x1148 + DIGI = bindings::PCI_VENDOR_ID_DIGI, // 0x114f + XIRCOM = bindings::PCI_VENDOR_ID_XIRCOM, // 0x115d + SERVERWORKS = bindings::PCI_VENDOR_ID_SERVERWORKS, // 0x1166 + ALTERA = bindings::PCI_VENDOR_ID_ALTERA, // 0x1172 + SBE = bindings::PCI_VENDOR_ID_SBE, // 0x1176 + TOSHIBA = bindings::PCI_VENDOR_ID_TOSHIBA, // 0x1179 + TOSHIBA_2 = bindings::PCI_VENDOR_ID_TOSHIBA_2, // 0x102f + ATTO = bindings::PCI_VENDOR_ID_ATTO, // 0x117c + RICOH = bindings::PCI_VENDOR_ID_RICOH, // 0x1180 + DLINK = bindings::PCI_VENDOR_ID_DLINK, // 0x1186 + ARTOP = bindings::PCI_VENDOR_ID_ARTOP, // 0x1191 + ZEITNET = bindings::PCI_VENDOR_ID_ZEITNET, // 0x1193 + FUJITSU_ME = bindings::PCI_VENDOR_ID_FUJITSU_ME, // 0x119e + MARVELL = bindings::PCI_VENDOR_ID_MARVELL, // 0x11ab + MARVELL_EXT = bindings::PCI_VENDOR_ID_MARVELL_EXT, // 0x1b4b + V3 = bindings::PCI_VENDOR_ID_V3, // 0x11b0 + ATT = bindings::PCI_VENDOR_ID_ATT, // 0x11c1 + SPECIALIX = bindings::PCI_VENDOR_ID_SPECIALIX, // 0x11cb + ANALOG_DEVICES = bindings::PCI_VENDOR_ID_ANALOG_DEVICES, // 0x11d4 + ZORAN = bindings::PCI_VENDOR_ID_ZORAN, // 0x11de + COMPEX = bindings::PCI_VENDOR_ID_COMPEX, // 0x11f6 + MICROSEMI = bindings::PCI_VENDOR_ID_MICROSEMI, // 0x11f8 + RP = bindings::PCI_VENDOR_ID_RP, // 0x11fe + CYCLADES = bindings::PCI_VENDOR_ID_CYCLADES, // 0x120e + O2 = bindings::PCI_VENDOR_ID_O2, // 0x1217 + THREEDX = bindings::PCI_VENDOR_ID_3DFX, // 0x121a + AVM = bindings::PCI_VENDOR_ID_AVM, // 0x1244 + STALLION = bindings::PCI_VENDOR_ID_STALLION, // 0x124d + AT = bindings::PCI_VENDOR_ID_AT, // 0x1259 + ASIX = bindings::PCI_VENDOR_ID_ASIX, // 0x125b + ESS = bindings::PCI_VENDOR_ID_ESS, // 0x125d + SATSAGEM = bindings::PCI_VENDOR_ID_SATSAGEM, // 0x1267 + ENSONIQ = bindings::PCI_VENDOR_ID_ENSONIQ, // 0x1274 + TRANSMETA = bindings::PCI_VENDOR_ID_TRANSMETA, // 0x1279 + ROCKWELL = bindings::PCI_VENDOR_ID_ROCKWELL, // 0x127A + ITE = bindings::PCI_VENDOR_ID_ITE, // 0x1283 + ALTEON = bindings::PCI_VENDOR_ID_ALTEON, // 0x12ae + NVIDIA_SGS = bindings::PCI_VENDOR_ID_NVIDIA_SGS, // 0x12d2 + PERICOM = bindings::PCI_VENDOR_ID_PERICOM, // 0x12D8 + AUREAL = bindings::PCI_VENDOR_ID_AUREAL, // 0x12eb + ELECTRONICDESIGNGMBH = bindings::PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, // 0x12f8 + ESDGMBH = bindings::PCI_VENDOR_ID_ESDGMBH, // 0x12fe + CB = bindings::PCI_VENDOR_ID_CB, // 0x1307 + SIIG = bindings::PCI_VENDOR_ID_SIIG, // 0x131f + RADISYS = bindings::PCI_VENDOR_ID_RADISYS, // 0x1331 + MICRO_MEMORY = bindings::PCI_VENDOR_ID_MICRO_MEMORY, // 0x1332 + DOMEX = bindings::PCI_VENDOR_ID_DOMEX, // 0x134a + INTASHIELD = bindings::PCI_VENDOR_ID_INTASHIELD, // 0x135a + QUATECH = bindings::PCI_VENDOR_ID_QUATECH, // 0x135C + SEALEVEL = bindings::PCI_VENDOR_ID_SEALEVEL, // 0x135e + HYPERCOPE = bindings::PCI_VENDOR_ID_HYPERCOPE, // 0x1365 + DIGIGRAM = bindings::PCI_VENDOR_ID_DIGIGRAM, // 0x1369 + KAWASAKI = bindings::PCI_VENDOR_ID_KAWASAKI, // 0x136b + CNET = bindings::PCI_VENDOR_ID_CNET, // 0x1371 + LMC = bindings::PCI_VENDOR_ID_LMC, // 0x1376 + NETGEAR = bindings::PCI_VENDOR_ID_NETGEAR, // 0x1385 + APPLICOM = bindings::PCI_VENDOR_ID_APPLICOM, // 0x1389 + MOXA = bindings::PCI_VENDOR_ID_MOXA, // 0x1393 + CCD = bindings::PCI_VENDOR_ID_CCD, // 0x1397 + EXAR = bindings::PCI_VENDOR_ID_EXAR, // 0x13a8 + MICROGATE = bindings::PCI_VENDOR_ID_MICROGATE, // 0x13c0 + THREEWARE = bindings::PCI_VENDOR_ID_3WARE, // 0x13C1 + IOMEGA = bindings::PCI_VENDOR_ID_IOMEGA, // 0x13ca + ABOCOM = bindings::PCI_VENDOR_ID_ABOCOM, // 0x13D1 + SUNDANCE = bindings::PCI_VENDOR_ID_SUNDANCE, // 0x13f0 + CMEDIA = bindings::PCI_VENDOR_ID_CMEDIA, // 0x13f6 + ADVANTECH = bindings::PCI_VENDOR_ID_ADVANTECH, // 0x13fe + MEILHAUS = bindings::PCI_VENDOR_ID_MEILHAUS, // 0x1402 + LAVA = bindings::PCI_VENDOR_ID_LAVA, // 0x1407 + TIMEDIA = bindings::PCI_VENDOR_ID_TIMEDIA, // 0x1409 + ICE = bindings::PCI_VENDOR_ID_ICE, // 0x1412 + MICROSOFT = bindings::PCI_VENDOR_ID_MICROSOFT, // 0x1414 + OXSEMI = bindings::PCI_VENDOR_ID_OXSEMI, // 0x1415 + CHELSIO = bindings::PCI_VENDOR_ID_CHELSIO, // 0x1425 + EDIMAX = bindings::PCI_VENDOR_ID_EDIMAX, // 0x1432 + ADLINK = bindings::PCI_VENDOR_ID_ADLINK, // 0x144a + SAMSUNG = bindings::PCI_VENDOR_ID_SAMSUNG, // 0x144d + GIGABYTE = bindings::PCI_VENDOR_ID_GIGABYTE, // 0x1458 + AMBIT = bindings::PCI_VENDOR_ID_AMBIT, // 0x1468 + MYRICOM = bindings::PCI_VENDOR_ID_MYRICOM, // 0x14c1 + MEDIATEK = bindings::PCI_VENDOR_ID_MEDIATEK, // 0x14c3 + TITAN = bindings::PCI_VENDOR_ID_TITAN, // 0x14D2 + PANACOM = bindings::PCI_VENDOR_ID_PANACOM, // 0x14d4 + SIPACKETS = bindings::PCI_VENDOR_ID_SIPACKETS, // 0x14d9 + AFAVLAB = bindings::PCI_VENDOR_ID_AFAVLAB, // 0x14db + AMPLICON = bindings::PCI_VENDOR_ID_AMPLICON, // 0x14dc + BCM_GVC = bindings::PCI_VENDOR_ID_BCM_GVC, // 0x14a4 + BROADCOM = bindings::PCI_VENDOR_ID_BROADCOM, // 0x14e4 + TOPIC = bindings::PCI_VENDOR_ID_TOPIC, // 0x151f + MAINPINE = bindings::PCI_VENDOR_ID_MAINPINE, // 0x1522 + ENE = bindings::PCI_VENDOR_ID_ENE, // 0x1524 + SYBA = bindings::PCI_VENDOR_ID_SYBA, // 0x1592 + MORETON = bindings::PCI_VENDOR_ID_MORETON, // 0x15aa + VMWARE = bindings::PCI_VENDOR_ID_VMWARE, // 0x15ad + ZOLTRIX = bindings::PCI_VENDOR_ID_ZOLTRIX, // 0x15b0 + MELLANOX = bindings::PCI_VENDOR_ID_MELLANOX, // 0x15b3 + DFI = bindings::PCI_VENDOR_ID_DFI, // 0x15bd + QUICKNET = bindings::PCI_VENDOR_ID_QUICKNET, // 0x15e2 + ADDIDATA = bindings::PCI_VENDOR_ID_ADDIDATA, // 0x15B8 + PDC = bindings::PCI_VENDOR_ID_PDC, // 0x15e9 + FARSITE = bindings::PCI_VENDOR_ID_FARSITE, // 0x1619 + ARIMA = bindings::PCI_VENDOR_ID_ARIMA, // 0x161f + BROCADE = bindings::PCI_VENDOR_ID_BROCADE, // 0x1657 + SIBYTE = bindings::PCI_VENDOR_ID_SIBYTE, // 0x166d + ATHEROS = bindings::PCI_VENDOR_ID_ATHEROS, // 0x168c + NETCELL = bindings::PCI_VENDOR_ID_NETCELL, // 0x169c + CENATEK = bindings::PCI_VENDOR_ID_CENATEK, // 0x16CA + SYNOPSYS = bindings::PCI_VENDOR_ID_SYNOPSYS, // 0x16c3 + USR = bindings::PCI_VENDOR_ID_USR, // 0x16ec + VITESSE = bindings::PCI_VENDOR_ID_VITESSE, // 0x1725 + LINKSYS = bindings::PCI_VENDOR_ID_LINKSYS, // 0x1737 + ALTIMA = bindings::PCI_VENDOR_ID_ALTIMA, // 0x173b + CAVIUM = bindings::PCI_VENDOR_ID_CAVIUM, // 0x177d + TECHWELL = bindings::PCI_VENDOR_ID_TECHWELL, // 0x1797 + BELKIN = bindings::PCI_VENDOR_ID_BELKIN, // 0x1799 + RDC = bindings::PCI_VENDOR_ID_RDC, // 0x17f3 + GLI = bindings::PCI_VENDOR_ID_GLI, // 0x17a0 + LENOVO = bindings::PCI_VENDOR_ID_LENOVO, // 0x17aa + QCOM = bindings::PCI_VENDOR_ID_QCOM, // 0x17cb + CDNS = bindings::PCI_VENDOR_ID_CDNS, // 0x17cd + ARECA = bindings::PCI_VENDOR_ID_ARECA, // 0x17d3 + S2IO = bindings::PCI_VENDOR_ID_S2IO, // 0x17d5 + SITECOM = bindings::PCI_VENDOR_ID_SITECOM, // 0x182d + TOPSPIN = bindings::PCI_VENDOR_ID_TOPSPIN, // 0x1867 + COMMTECH = bindings::PCI_VENDOR_ID_COMMTECH, // 0x18f7 + SILAN = bindings::PCI_VENDOR_ID_SILAN, // 0x1904 + RENESAS = bindings::PCI_VENDOR_ID_RENESAS, // 0x1912 + SOLARFLARE = bindings::PCI_VENDOR_ID_SOLARFLARE, // 0x1924 + TDI = bindings::PCI_VENDOR_ID_TDI, // 0x192E + NXP = bindings::PCI_VENDOR_ID_NXP, // 0x1957 + PASEMI = bindings::PCI_VENDOR_ID_PASEMI, // 0x1959 + ATTANSIC = bindings::PCI_VENDOR_ID_ATTANSIC, // 0x1969 + JMICRON = bindings::PCI_VENDOR_ID_JMICRON, // 0x197B + KORENIX = bindings::PCI_VENDOR_ID_KORENIX, // 0x1982 + HUAWEI = bindings::PCI_VENDOR_ID_HUAWEI, // 0x19e5 + NETRONOME = bindings::PCI_VENDOR_ID_NETRONOME, // 0x19ee + QMI = bindings::PCI_VENDOR_ID_QMI, // 0x1a32 + AZWAVE = bindings::PCI_VENDOR_ID_AZWAVE, // 0x1a3b + REDHAT_QUMRANET = bindings::PCI_VENDOR_ID_REDHAT_QUMRANET, // 0x1af4 + ASMEDIA = bindings::PCI_VENDOR_ID_ASMEDIA, // 0x1b21 + REDHAT = bindings::PCI_VENDOR_ID_REDHAT, // 0x1b36 + WCHIC = bindings::PCI_VENDOR_ID_WCHIC, // 0x1c00 + SILICOM_DENMARK = bindings::PCI_VENDOR_ID_SILICOM_DENMARK, // 0x1c2c + AMAZON_ANNAPURNA_LABS = bindings::PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, // 0x1c36 + CIRCUITCO = bindings::PCI_VENDOR_ID_CIRCUITCO, // 0x1cc8 + AMAZON = bindings::PCI_VENDOR_ID_AMAZON, // 0x1d0f + ZHAOXIN = bindings::PCI_VENDOR_ID_ZHAOXIN, // 0x1d17 + ROCKCHIP = bindings::PCI_VENDOR_ID_ROCKCHIP, // 0x1d87 + HYGON = bindings::PCI_VENDOR_ID_HYGON, // 0x1d94 + META = bindings::PCI_VENDOR_ID_META, // 0x1d9b + FUNGIBLE = bindings::PCI_VENDOR_ID_FUNGIBLE, // 0x1dad + HXT = bindings::PCI_VENDOR_ID_HXT, // 0x1dbf + TEKRAM = bindings::PCI_VENDOR_ID_TEKRAM, // 0x1de1 + RPI = bindings::PCI_VENDOR_ID_RPI, // 0x1de4 + ALIBABA = bindings::PCI_VENDOR_ID_ALIBABA, // 0x1ded + CXL = bindings::PCI_VENDOR_ID_CXL, // 0x1e98 + TEHUTI = bindings::PCI_VENDOR_ID_TEHUTI, // 0x1fc9 + SUNIX = bindings::PCI_VENDOR_ID_SUNIX, // 0x1fd4 + HINT = bindings::PCI_VENDOR_ID_HINT, // 0x3388 + THREEDLABS = bindings::PCI_VENDOR_ID_3DLABS, // 0x3d3d + NETXEN = bindings::PCI_VENDOR_ID_NETXEN, // 0x4040 + AKS = bindings::PCI_VENDOR_ID_AKS, // 0x416c + WCHCN = bindings::PCI_VENDOR_ID_WCHCN, // 0x4348 + ACCESSIO = bindings::PCI_VENDOR_ID_ACCESSIO, // 0x494f + S3 = bindings::PCI_VENDOR_ID_S3, // 0x5333 + DUNORD = bindings::PCI_VENDOR_ID_DUNORD, // 0x5544 + DCI = bindings::PCI_VENDOR_ID_DCI, // 0x6666 + GLENFLY = bindings::PCI_VENDOR_ID_GLENFLY, // 0x6766 + INTEL = bindings::PCI_VENDOR_ID_INTEL, // 0x8086 + WANGXUN = bindings::PCI_VENDOR_ID_WANGXUN, // 0x8088 + SCALEMP = bindings::PCI_VENDOR_ID_SCALEMP, // 0x8686 + COMPUTONE = bindings::PCI_VENDOR_ID_COMPUTONE, // 0x8e0e + KTI = bindings::PCI_VENDOR_ID_KTI, // 0x8e2e + ADAPTEC = bindings::PCI_VENDOR_ID_ADAPTEC, // 0x9004 + ADAPTEC2 = bindings::PCI_VENDOR_ID_ADAPTEC2, // 0x9005 + HOLTEK = bindings::PCI_VENDOR_ID_HOLTEK, // 0x9412 + NETMOS = bindings::PCI_VENDOR_ID_NETMOS, // 0x9710 + THREECOM_2 = bindings::PCI_VENDOR_ID_3COM_2, // 0xa727 + SOLIDRUN = bindings::PCI_VENDOR_ID_SOLIDRUN, // 0xd063 + DIGIUM = bindings::PCI_VENDOR_ID_DIGIUM, // 0xd161 + TIGERJET = bindings::PCI_VENDOR_ID_TIGERJET, // 0xe159 + XILINX_RME = bindings::PCI_VENDOR_ID_XILINX_RME, // 0xea60 + XEN = bindings::PCI_VENDOR_ID_XEN, // 0x5853 + OCZ = bindings::PCI_VENDOR_ID_OCZ, // 0x1b85 + NCUBE = bindings::PCI_VENDOR_ID_NCUBE, // 0x10ff +} diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs new file mode 100644 index 000000000..953e16735 --- /dev/null +++ b/rust/kernel/pci/io.rs @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI memory-mapped I/O infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + devres::DevresLt, + io::{ + IoBackend, + IoBase, + IoCapable, + Mmio, + MmioBackend, + MmioRaw, + Region, // + }, + prelude::*, + ptr::KnownSize, + types::{ + CovariantForLt, + ForLt, // + }, // +}; + +/// Represents the size of a PCI configuration space. +/// +/// PCI devices can have either a *normal* (legacy) configuration space of 256 bytes, +/// or an *extended* configuration space of 4096 bytes as defined in the PCI Express +/// specification. +#[repr(usize)] +#[derive(Eq, PartialEq)] +pub enum ConfigSpaceSize { + /// 256-byte legacy PCI configuration space. + Normal = 256, + + /// 4096-byte PCIe extended configuration space. + Extended = 4096, +} + +impl ConfigSpaceSize { + /// Get the raw value of this enum. + #[inline(always)] + pub const fn into_raw(self) -> usize { + // CAST: PCI configuration space size is at most 4096 bytes, so the value always fits + // within `usize` without truncation or sign change. + self as usize + } +} + +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; + +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; + +/// A view of PCI configuration space of a device. +/// +/// Provides typed read and write accessors for configuration registers +/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. +/// +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). +/// +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { + pub(crate) pdev: &'a Device, + ptr: *mut T, +} + +impl Copy for ConfigSpace<'_, T> {} +impl Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for ConfigSpace<'_, T> {} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } +} + +/// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_config_space_io_capable { + ($ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + + let mut val: $ty = 0; + + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; + val + } + + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; + } + } + }; +} + +// PCI configuration space supports 8, 16, and 32-bit accesses. +impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); +impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); +impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ConfigSpace<'a, T> { + self + } +} + +/// A PCI BAR to perform I/O-Operations on. +/// +/// I/O backend assumes that the device is little-endian and will automatically +/// convert from little-endian to CPU endianness. +/// +/// # Invariants +/// +/// `Bar` always holds an `IoRaw` instance that holds a valid pointer to the start of the I/O +/// memory mapped PCI BAR and its size. +pub struct Bar<'a, const SIZE: usize = 0> { + pdev: &'a Device, + io: MmioRaw>, + num: i32, +} + +impl ForLt for Bar<'static, SIZE> { + type Of<'a> = Bar<'a, SIZE>; +} + +// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for Bar<'static, SIZE> {} + +/// A device-managed PCI BAR mapping. +/// +/// See [`Bar::into_devres`]. +pub type DevresBar = DevresLt>; + +impl<'a, const SIZE: usize> Bar<'a, SIZE> { + pub(super) fn new( + pdev: &'a Device, + num: u32, + name: &'static CStr, + ) -> Result { + let len = pdev.resource_len(num)?; + if len == 0 { + return Err(ENOMEM); + } + + // Convert to `i32`, since that's what all the C bindings use. + let num = i32::try_from(num)?; + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) }; + if ret != 0 { + return Err(EBUSY); + } + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize; + if ioptr == 0 { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { bindings::pci_release_region(pdev.as_raw(), num) }; + return Err(ENOMEM); + } + + let io = match MmioRaw::new_region(ioptr, len as usize) { + Ok(io) => io, + Err(err) => { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { Self::do_release(pdev, ioptr, num) }; + return Err(err); + } + }; + + Ok(Bar { pdev, io, num }) + } + + /// # Safety + /// + /// `ioptr` must be a valid pointer to the memory mapped PCI BAR number `num`. + unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is valid by the safety requirements. + // `num` is valid by the safety requirements. + unsafe { + bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut c_void); + bindings::pci_release_region(pdev.as_raw(), num); + } + } + + fn release(&self) { + // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. + unsafe { Self::do_release(self.pdev, self.io.addr(), self.num) }; + } + + /// Consume the `Bar` and register it as a device-managed resource. + /// + /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data. + /// Access to the BAR is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let pdev = self.pdev; + // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(pdev.as_ref(), self) } + } +} + +impl Bar<'_> { + #[inline] + pub(super) fn index_is_valid(index: u32) -> bool { + // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. + index < bindings::PCI_NUM_RESOURCES + } +} + +impl Drop for Bar<'_, SIZE> { + fn drop(&mut self) { + self.release(); + } +} + +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region; + + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. + unsafe { Mmio::from_raw(self.io) } + } +} + +impl Device { + /// Maps an entire PCI BAR after performing a region-request on it. I/O operation bound checks + /// can be performed on compile time for offsets (plus the requested type size) < SIZE. + pub fn iomap_region_sized<'a, const SIZE: usize>( + &'a self, + bar: u32, + name: &'static CStr, + ) -> Result> { + Bar::new(self, bar, name) + } + + /// Maps an entire PCI BAR after performing a region-request on it. + pub fn iomap_region<'a>(&'a self, bar: u32, name: &'static CStr) -> Result> { + self.iomap_region_sized::<0>(bar, name) + } + + /// Returns the size of configuration space. + pub fn cfg_size(&self) -> ConfigSpaceSize { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + let size = unsafe { (*self.as_raw()).cfg_size }; + match size { + 256 => ConfigSpaceSize::Normal, + 4096 => ConfigSpaceSize::Extended, + _ => { + // PANIC: The PCI subsystem only ever reports the configuration space size as either + // `ConfigSpaceSize::Normal` or `ConfigSpaceSize::Extended`. + unreachable!(); + } + } + } + + /// Return a view of the normal (256-byte) config space. + pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. + ConfigSpace { + pdev: self, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), + } + } + + /// Return a view of the extended (4096-byte) config space. + pub fn config_space_extended<'a>(&'a self) -> Result> { + if self.cfg_size() != ConfigSpaceSize::Extended { + return Err(EINVAL); + } + + // INVARIANT: null is aligned and we just checked the `cfg_size`. + Ok(ConfigSpace { + pdev: self, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), + }) + } +} diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs new file mode 100644 index 000000000..22e2cdf82 --- /dev/null +++ b/rust/kernel/pci/irq.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI interrupt infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + device::Bound, + error::to_result, + irq::IrqRequest, + prelude::*, // +}; +use core::num::NonZero; + +/// IRQ type flags for PCI interrupt allocation. +#[derive(Debug, Clone, Copy)] +pub enum IrqType { + /// INTx interrupts. + Intx, + /// Message Signaled Interrupts (MSI). + Msi, + /// Extended Message Signaled Interrupts (MSI-X). + MsiX, +} + +impl IrqType { + /// Convert to the corresponding kernel flags. + const fn as_raw(self) -> u32 { + match self { + IrqType::Intx => bindings::PCI_IRQ_INTX, + IrqType::Msi => bindings::PCI_IRQ_MSI, + IrqType::MsiX => bindings::PCI_IRQ_MSIX, + } + } + + /// Construct from raw value. + #[inline] + const fn from_raw(raw: u32) -> Self { + match raw { + bindings::PCI_IRQ_MSIX => IrqType::MsiX, + bindings::PCI_IRQ_MSI => IrqType::Msi, + _ => IrqType::Intx, + } + } +} + +/// Set of IRQ types that can be used for PCI interrupt allocation. +#[derive(Debug, Clone, Copy, Default)] +pub struct IrqTypes(u32); + +impl IrqTypes { + /// Create a set containing all IRQ types (MSI-X, MSI, and INTx). + pub const fn all() -> Self { + Self(bindings::PCI_IRQ_ALL_TYPES) + } + + /// Build a set of IRQ types. + /// + /// # Examples + /// + /// ```ignore + /// // Create a set with only MSI and MSI-X (no INTx interrupts). + /// let msi_only = IrqTypes::default() + /// .with(IrqType::Msi) + /// .with(IrqType::MsiX); + /// ``` + pub const fn with(self, irq_type: IrqType) -> Self { + Self(self.0 | irq_type.as_raw()) + } + + /// Get the raw flags value. + const fn as_raw(self) -> u32 { + self.0 + } +} + +/// A resolved IRQ vector from a PCI interrupt vector allocation. +/// +/// Created by [`IrqVectorRegistration::index`]. Convert to [`IrqRequest`] via [`From`] to register +/// a handler with [`irq::Registration::new`](crate::irq::Registration::new). +pub struct IrqVector<'a> { + request: IrqRequest<'a>, + reg: &'a IrqVectorRegistration<'a>, +} + +impl<'a> IrqVector<'a> { + /// Creates a new [`IrqVector`] with an already resolved [`IrqRequest`]. + /// + /// # Safety + /// + /// `request` must have been resolved from `reg`. + #[inline] + unsafe fn new(request: IrqRequest<'a>, reg: &'a IrqVectorRegistration<'a>) -> Self { + Self { request, reg } + } + + /// Returns the [`IrqVectorRegistration`] this vector was derived from. + #[inline] + pub fn vectors(&self) -> &'a IrqVectorRegistration<'a> { + self.reg + } + + /// Returns the interrupt type the PCI core selected for this vector's allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + self.reg.irq_type() + } +} + +impl<'a> From> for IrqRequest<'a> { + #[inline] + fn from(vector: IrqVector<'a>) -> Self { + vector.request + } +} + +/// An allocation of PCI interrupt vectors for a device. +/// +/// This type owns the vector allocation; dropping it frees the vectors. IRQ handlers borrow from +/// this registration and must be dropped before it is. +/// +/// # Invariants +/// +/// `dev` has an allocation of `len` interrupt vectors. +pub struct IrqVectorRegistration<'a> { + dev: &'a Device, + len: NonZero, +} + +impl<'a> IrqVectorRegistration<'a> { + /// Returns the number of allocated vectors. + /// + /// This is at least the `min_vecs` that [`Device::alloc_irq_vectors`] was asked for. + #[inline] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len.get() + } + + /// Returns the interrupt type the PCI core selected for this allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + IrqType::from_raw(unsafe { bindings::pci_irq_type(self.dev.as_raw()) }) + } + + /// Returns the [`IrqVector`] at `index`. + /// + /// Returns [`EINVAL`] if the `index` is out of bounds for the length reported by + /// [`Self::len()`]. + #[inline] + pub fn index(&self, index: usize) -> Result> { + let index = u32::try_from(index)?; + + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index) }; + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is a valid IRQ number for `self.dev`, resolved from this registration. + Ok(unsafe { IrqVector::new(IrqRequest::new(self.dev.as_ref(), irq as u32), self) }) + } +} + +impl Drop for IrqVectorRegistration<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.dev.as_raw()` is a valid pointer to a + // `struct pci_dev` that has successfully allocated IRQ vectors. + unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; + } +} + +impl Device { + /// Allocate IRQ vectors for this PCI device. + /// + /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. + /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` + /// parameter and hardware capabilities. When multiple types are specified, the kernel + /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. + /// + /// The allocated vectors are freed when the returned [`IrqVectorRegistration`] is dropped. + /// Use [`IrqVectorRegistration::index`] to obtain an [`IrqVector`] for a given vector + /// index. + /// + /// # Arguments + /// + /// * `min_vecs` - Minimum number of vectors required. + /// * `max_vecs` - Maximum number of vectors to allocate. + /// * `irq_types` - Types of interrupts that can be used. + /// + /// # Returns + /// + /// Returns the IRQ vector registration, or an error if `min_vecs` vectors cannot be + /// allocated. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{ device::Bound, pci}; + /// # fn no_run(dev: &pci::Device) -> Result { + /// // Allocate using any available interrupt type in the order mentioned above. + /// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?; + /// + /// // Allocate MSI or MSI-X only (no INTx interrupts). + /// let msi_only = pci::IrqTypes::default() + /// .with(pci::IrqType::Msi) + /// .with(pci::IrqType::MsiX); + /// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?; + /// # Ok(()) + /// # } + /// ``` + pub fn alloc_irq_vectors( + &self, + min_vecs: u32, + max_vecs: u32, + irq_types: IrqTypes, + ) -> Result> { + // SAFETY: + // - `self.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + to_result(ret)?; + + let len = NonZero::new(ret as usize).ok_or(EINVAL)?; + + // INVARIANT: `pci_alloc_irq_vectors()` allocated `len` vectors for `self`. + Ok(IrqVectorRegistration { dev: self, len }) + } +} diff --git a/rust/kernel/pid_namespace.rs b/rust/kernel/pid_namespace.rs new file mode 100644 index 000000000..979a9718f --- /dev/null +++ b/rust/kernel/pid_namespace.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (c) 2024 Christian Brauner + +//! Pid namespaces. +//! +//! C header: [`include/linux/pid_namespace.h`](srctree/include/linux/pid_namespace.h) and +//! [`include/linux/pid.h`](srctree/include/linux/pid.h) + +use crate::{bindings, sync::aref::AlwaysRefCounted, types::Opaque}; +use core::ptr; + +/// Wraps the kernel's `struct pid_namespace`. Thread safe. +/// +/// This structure represents the Rust abstraction for a C `struct pid_namespace`. This +/// implementation abstracts the usage of an already existing C `struct pid_namespace` within Rust +/// code that we get passed from the C side. +#[repr(transparent)] +pub struct PidNamespace { + inner: Opaque, +} + +impl PidNamespace { + /// Returns a raw pointer to the inner C struct. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::pid_namespace { + self.inner.get() + } + + /// Creates a reference to a [`PidNamespace`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`PidNamespace`] reference. + pub unsafe fn from_ptr<'a>(ptr: *const bindings::pid_namespace) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `PidNamespace` type being transparent makes the cast ok. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: Instances of `PidNamespace` are always reference-counted. +unsafe impl AlwaysRefCounted for PidNamespace { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_pid_ns(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_pid_ns(obj.cast().as_ptr()) } + } +} + +// SAFETY: +// - `PidNamespace::dec_ref` can be called from any thread. +// - It is okay to send ownership of `PidNamespace` across thread boundaries. +unsafe impl Send for PidNamespace {} + +// SAFETY: It's OK to access `PidNamespace` through shared references from other threads because +// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for PidNamespace {} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs new file mode 100644 index 000000000..ac0a012ae --- /dev/null +++ b/rust/kernel/platform.rs @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the platform bus. +//! +//! C header: [`include/linux/platform_device.h`](srctree/include/linux/platform_device.h) + +use crate::{ + acpi, + bindings, + container_of, + device::{ + self, + Bound, // + }, + driver, + error::{ + from_result, + to_result, // + }, + io::Resource, + irq::{ + self, + IrqRequest, // + }, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; + +#[cfg(CONFIG_HAS_IOMEM)] +use crate::io::mem::IoRequest; + +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::{ + addr_of_mut, + NonNull, // + }, +}; + +/// An adapter for the registration of platform drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::platform_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct platform_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::platform_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + pdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct platform_driver` on initialization. + unsafe { + (*pdrv.get()).driver.name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).driver.of_match_table = of_table; + (*pdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::__platform_driver_register(pdrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(pdrv: &Opaque) { + // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::platform_driver_unregister(pdrv.get()) }; + } +} + +impl Adapter { + extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int { + // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + // SAFETY: `pdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(pdev.as_ref()) }; + + from_result(|| { + let data = T::probe(pdev, info); + + pdev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { + // SAFETY: The platform bus only ever calls the remove callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; + + T::unbind(pdev, data); + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single platform driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_platform_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_platform_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::platform::Adapter, { $($f)* }); + }; +} + +/// The platform driver trait. +/// +/// Drivers must implement this trait in order to get a platform driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// # acpi, +/// # bindings, +/// # device::Core, +/// # of, +/// # platform, +/// # }; +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// impl platform::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// _pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// } +///``` +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// Platform driver probe. + /// + /// Called when a new platform device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + dev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (dev, this); + } +} + +/// The platform device representation. +/// +/// This structure represents the Rust abstraction for a C `struct platform_device`. The +/// implementation abstracts the usage of an already existing C `struct platform_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct platform_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::platform_device { + self.0.get() + } + + /// Returns the resource at `index`, if any. + pub fn resource_by_index(&self, index: u32) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct platform_device`. + let resource = unsafe { + bindings::platform_get_resource(self.as_raw(), bindings::IORESOURCE_MEM, index) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } + + /// Returns the resource with a given `name`, if any. + pub fn resource_by_name(&self, name: &CStr) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct + // platform_device` and `name` points to a valid C string. + let resource = unsafe { + bindings::platform_get_resource_byname( + self.as_raw(), + bindings::IORESOURCE_MEM, + name.as_char_ptr(), + ) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } +} + +#[cfg(CONFIG_HAS_IOMEM)] +impl Device { + /// Returns an `IoRequest` for the resource at `index`, if any. + pub fn io_request_by_index(&self, index: u32) -> Option> { + self.resource_by_index(index) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } + + /// Returns an `IoRequest` for the resource with a given `name`, if any. + pub fn io_request_by_name(&self, name: &CStr) -> Option> { + self.resource_by_name(name) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } +} + +// SAFETY: `platform::Device` is a transparent wrapper of `struct platform_device`. +// The offset is guaranteed to point to a valid device field inside `platform::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::platform_device, dev); +} + +macro_rules! define_irq_accessor_by_index { + ( + $(#[$meta:meta])* $fn_name:ident, + $request_fn:ident, + $reg_type:ident, + $handler_trait:ident + ) => { + $(#[$meta])* + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( + &'a self, + flags: irq::Flags, + index: u32, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(index)?; + + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) + }) + } + }; +} + +macro_rules! define_irq_accessor_by_name { + ( + $(#[$meta:meta])* $fn_name:ident, + $request_fn:ident, + $reg_type:ident, + $handler_trait:ident + ) => { + $(#[$meta])* + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( + &'a self, + flags: irq::Flags, + irq_name: &'a CStr, + name: &'static CStr, + handler: impl PinInit + 'a, + ) -> impl PinInit, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(irq_name)?; + + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) + }) + } + }; +} + +impl Device { + /// Returns an [`IrqRequest`] for the IRQ at the given index, if any. + pub fn irq_by_index(&self, index: u32) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq(self.as_raw(), index) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ at the given index, but does not + /// print an error if the IRQ cannot be obtained. + pub fn optional_irq_by_index(&self, index: u32) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq_optional(self.as_raw(), index) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ with the given name, if any. + pub fn irq_by_name(&self, name: &CStr) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { bindings::platform_get_irq_byname(self.as_raw(), name.as_char_ptr()) }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + /// Returns an [`IrqRequest`] for the IRQ with the given name, but does not + /// print an error if the IRQ cannot be obtained. + pub fn optional_irq_by_name(&self, name: &CStr) -> Result> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct platform_device`. + let irq = unsafe { + bindings::platform_get_irq_byname_optional(self.as_raw(), name.as_char_ptr()) + }; + + if irq < 0 { + return Err(Error::from_errno(irq)); + } + + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) + } + + define_irq_accessor_by_index!( + /// Returns a [`irq::Registration`] for the IRQ at the given index. + request_irq_by_index, + irq_by_index, + Registration, + Handler + ); + define_irq_accessor_by_name!( + /// Returns a [`irq::Registration`] for the IRQ with the given name. + request_irq_by_name, + irq_by_name, + Registration, + Handler + ); + define_irq_accessor_by_index!( + /// Does the same as [`Self::request_irq_by_index`], except that it does + /// not print an error message if the IRQ cannot be obtained. + request_optional_irq_by_index, + optional_irq_by_index, + Registration, + Handler + ); + define_irq_accessor_by_name!( + /// Does the same as [`Self::request_irq_by_name`], except that it does + /// not print an error message if the IRQ cannot be obtained. + request_optional_irq_by_name, + optional_irq_by_name, + Registration, + Handler + ); + + define_irq_accessor_by_index!( + /// Returns a [`irq::ThreadedRegistration`] for the IRQ at the given index. + request_threaded_irq_by_index, + irq_by_index, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_name!( + /// Returns a [`irq::ThreadedRegistration`] for the IRQ with the given name. + request_threaded_irq_by_name, + irq_by_name, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_index!( + /// Does the same as [`Self::request_threaded_irq_by_index`], except + /// that it does not print an error message if the IRQ cannot be + /// obtained. + request_optional_threaded_irq_by_index, + optional_irq_by_index, + ThreadedRegistration, + ThreadedHandler + ); + define_irq_accessor_by_name!( + /// Does the same as [`Self::request_threaded_irq_by_name`], except that + /// it does not print an error message if the IRQ cannot be obtained. + request_optional_threaded_irq_by_name, + optional_irq_by_name, + ThreadedRegistration, + ThreadedHandler + ); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +impl<'a> crate::dma::Device<'a> for Device> {} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::platform_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct platform_device`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl TryFrom<&device::Device> for &Device { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device) -> Result { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if !unsafe { bindings::dev_is_platform(dev.as_raw()) } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the bus type of `dev` equals + // `bindings::platform_bus_type`, hence `dev` must be embedded in a valid + // `struct platform_device` as guaranteed by the corresponding C code. + let pdev = unsafe { container_of!(dev.as_raw(), bindings::platform_device, dev) }; + + // SAFETY: `pdev` is a valid pointer to a `struct platform_device`. + Ok(unsafe { &*pdev.cast() }) + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct platform_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs new file mode 100644 index 000000000..ca396f1f7 --- /dev/null +++ b/rust/kernel/prelude.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` prelude. +//! +//! These are the most common items used by Rust code in the kernel, +//! intended to be imported by all Rust code, for convenience. +//! +//! # Examples +//! +//! ``` +//! use kernel::prelude::*; +//! ``` + +#[doc(no_inline)] +pub use core::{ + mem::{ + align_of, + align_of_val, + size_of, + size_of_val, // + }, + pin::Pin, // +}; + +#[doc(no_inline)] +pub use ::ffi::{ + c_char, + c_int, + c_long, + c_longlong, + c_schar, + c_short, + c_uchar, + c_uint, + c_ulong, + c_ulonglong, + c_ushort, + c_void, + CStr, // +}; + +#[doc(no_inline)] +pub use macros::{ + export, + fmt, + kunit_tests, + module, + vtable, // +}; + +#[doc(no_inline)] +pub use pin_init::{ + init, + pin_data, + pin_init, + pinned_drop, + InPlaceWrite, + Init, + PinInit, + Zeroable, // +}; + +#[doc(no_inline)] +pub use zerocopy::{ + FromBytes, + IntoBytes, // +}; + +#[doc(no_inline)] +pub use zerocopy_derive::{ + FromBytes, + IntoBytes, // +}; + +#[doc(no_inline)] +pub use super::{ + alloc::{ + flags::*, + Box, + KBox, + KVBox, + KVVec, + KVec, + VBox, + VVec, + Vec, // + }, + build_assert::{ + build_assert, + build_error, + const_assert, + static_assert, // + }, + current, + dev_alert, + dev_crit, + dev_dbg, + dev_emerg, + dev_err, + dev_info, + dev_notice, + dev_warn, + error::{ + code::*, + Error, + Result, // + }, + init::InPlaceInit, + pr_alert, + pr_crit, + pr_debug, + pr_emerg, + pr_err, + pr_info, + pr_notice, + pr_warn, + str::CStrExt as _, + try_init, + try_pin_init, + uaccess::UserPtr, + ThisModule, // +}; + +// `super::std_vendor` is hidden, which makes the macro inline for some reason. +#[doc(no_inline)] +pub use super::dbg; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs new file mode 100644 index 000000000..0d62beeed --- /dev/null +++ b/rust/kernel/print.rs @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Printing facilities. +//! +//! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h) +//! +//! Reference: + +use crate::{ + ffi::{c_char, c_void}, + fmt, + prelude::*, + str::RawFormatter, + sync::atomic::{ + Atomic, + AtomicType, + Relaxed, // + }, +}; + +// Called from `vsprintf` with format specifier `%pA`. +#[expect(clippy::missing_safety_doc)] +#[export] +unsafe extern "C" fn rust_fmt_argument( + buf: *mut c_char, + end: *mut c_char, + ptr: *const c_void, +) -> *mut c_char { + use fmt::Write; + // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`. + let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) }; + // SAFETY: TODO. + let _ = w.write_fmt(unsafe { *ptr.cast::>() }); + w.pos().cast() +} + +/// Format strings. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +pub mod format_strings { + /// The length we copy from the `KERN_*` kernel prefixes. + const LENGTH_PREFIX: usize = 2; + + /// The length of the fixed format strings. + pub const LENGTH: usize = 10; + + /// Generates a fixed format string for the kernel's [`_printk`]. + /// + /// The format string is always the same for a given level, i.e. for a + /// given `prefix`, which are the kernel's `KERN_*` constants. + /// + /// [`_printk`]: srctree/include/linux/printk.h + const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { + // Ensure the `KERN_*` macros are what we expect. + assert!(prefix[0] == b'\x01'); + if is_cont { + assert!(prefix[1] == b'c'); + } else { + assert!(prefix[1] >= b'0' && prefix[1] <= b'7'); + } + assert!(prefix[2] == b'\x00'); + + let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont { + b"%pA\0\0\0\0\0" + } else { + b"%s: %pA\0" + }; + + [ + prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5], + suffix[6], suffix[7], + ] + } + + // Generate the format strings at compile-time. + // + // This avoids the compiler generating the contents on the fly in the stack. + // + // Furthermore, `static` instead of `const` is used to share the strings + // for all the kernel. + pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG); + pub static ALERT: [u8; LENGTH] = generate(false, bindings::KERN_ALERT); + pub static CRIT: [u8; LENGTH] = generate(false, bindings::KERN_CRIT); + pub static ERR: [u8; LENGTH] = generate(false, bindings::KERN_ERR); + pub static WARNING: [u8; LENGTH] = generate(false, bindings::KERN_WARNING); + pub static NOTICE: [u8; LENGTH] = generate(false, bindings::KERN_NOTICE); + pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO); + pub static DEBUG: [u8; LENGTH] = generate(false, bindings::KERN_DEBUG); + pub static CONT: [u8; LENGTH] = generate(true, bindings::KERN_CONT); +} + +/// Prints a message via the kernel's [`_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// # Safety +/// +/// The format string must be one of the ones in [`format_strings`], and +/// the module name must be null-terminated. +/// +/// [`_printk`]: srctree/include/linux/printk.h +#[doc(hidden)] +#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] +pub unsafe fn call_printk( + format_string: &[u8; format_strings::LENGTH], + module_name: &[u8], + args: fmt::Arguments<'_>, +) { + // `_printk` does not seem to fail in any path. + #[cfg(CONFIG_PRINTK)] + // SAFETY: TODO. + unsafe { + bindings::_printk( + format_string.as_ptr(), + module_name.as_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Prints a message via the kernel's [`_printk`] for the `CONT` level. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// [`_printk`]: srctree/include/linux/printk.h +#[doc(hidden)] +#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] +pub fn call_printk_cont(args: fmt::Arguments<'_>) { + // `_printk` does not seem to fail in any path. + // + // SAFETY: The format string is fixed. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + format_strings::CONT.as_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } +} + +/// Performs formatting and forwards the string to [`call_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +#[cfg(not(testlib))] +#[macro_export] +#[expect(clippy::crate_in_macro_def)] +macro_rules! print_macro ( + // The non-continuation cases (most of them, e.g. `INFO`). + ($format_string:path, false, $($arg:tt)+) => ( + // To remain sound, `arg`s must be expanded outside the `unsafe` block. + // Typically one would use a `let` binding for that; however, `format_args!` + // takes borrows on the arguments, but does not extend the scope of temporaries. + // Therefore, a `match` expression is used to keep them around, since + // the scrutinee is kept until the end of the `match`. + match $crate::prelude::fmt!($($arg)+) { + // SAFETY: This hidden macro should only be called by the documented + // printing macros which ensure the format string is one of the fixed + // ones. All `__LOG_PREFIX`s are null-terminated as they are generated + // by the `module!` proc macro or fixed values defined in a kernel + // crate. + args => unsafe { + $crate::print::call_printk( + &$format_string, + crate::__LOG_PREFIX, + args, + ); + } + } + ); + + // The `CONT` case. + ($format_string:path, true, $($arg:tt)+) => ( + $crate::print::call_printk_cont( + $crate::prelude::fmt!($($arg)+), + ); + ); +); + +/// Stub for doctests +#[cfg(testlib)] +#[macro_export] +macro_rules! print_macro ( + ($format_string:path, $e:expr, $($arg:tt)+) => ( + () + ); +); + +// We could use a macro to generate these macros. However, doing so ends +// up being a bit ugly: it requires the dollar token trick to escape `$` as +// well as playing with the `doc` attribute. Furthermore, they cannot be easily +// imported in the prelude due to [1]. So, for the moment, we just write them +// manually, like in the C side; while keeping most of the logic in another +// macro, i.e. [`print_macro`]. +// +// [1]: https://github.com/rust-lang/rust/issues/52234 + +/// Prints an emergency-level message (level 0). +/// +/// Use this level if the system is unusable. +/// +/// Equivalent to the kernel's [`pr_emerg`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_emerg`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_emerg +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_emerg!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_emerg ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::EMERG, false, $($arg)*) + ) +); + +/// Prints an alert-level message (level 1). +/// +/// Use this level if action must be taken immediately. +/// +/// Equivalent to the kernel's [`pr_alert`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_alert`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_alert +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_alert!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_alert ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ALERT, false, $($arg)*) + ) +); + +/// Prints a critical-level message (level 2). +/// +/// Use this level for critical conditions. +/// +/// Equivalent to the kernel's [`pr_crit`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_crit`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_crit +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_crit!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_crit ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CRIT, false, $($arg)*) + ) +); + +/// Prints an error-level message (level 3). +/// +/// Use this level for error conditions. +/// +/// Equivalent to the kernel's [`pr_err`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_err`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_err +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_err!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_err ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ERR, false, $($arg)*) + ) +); + +/// Prints a warning-level message (level 4). +/// +/// Use this level for warning conditions. +/// +/// Equivalent to the kernel's [`pr_warn`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_warn`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_warn +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_warn!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_warn ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::WARNING, false, $($arg)*) + ) +); + +/// Prints a notice-level message (level 5). +/// +/// Use this level for normal but significant conditions. +/// +/// Equivalent to the kernel's [`pr_notice`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_notice`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_notice +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_notice!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_notice ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::NOTICE, false, $($arg)*) + ) +); + +/// Prints an info-level message (level 6). +/// +/// Use this level for informational messages. +/// +/// Equivalent to the kernel's [`pr_info`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_info`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_info +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_info!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_info ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::INFO, false, $($arg)*) + ) +); + +/// Prints a debug-level message (level 7). +/// +/// Use this level for debug messages. +/// +/// Equivalent to the kernel's [`pr_debug`] macro, except that it doesn't support dynamic debug +/// yet. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_debug`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_debug +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// pr_debug!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_debug ( + ($($arg:tt)*) => ( + if cfg!(debug_assertions) { + $crate::print_macro!($crate::print::format_strings::DEBUG, false, $($arg)*) + } + ) +); + +/// Continues a previous log message in the same line. +/// +/// Use only when continuing a previous `pr_*!` macro (e.g. [`pr_info!`]). +/// +/// Equivalent to the kernel's [`pr_cont`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`std::format!`] for information about the formatting syntax. +/// +/// [`pr_info!`]: crate::pr_info! +/// [`pr_cont`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_cont +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::pr_cont; +/// pr_info!("hello"); +/// pr_cont!(" {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_cont ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CONT, true, $($arg)*) + ) +); + +/// A lightweight `call_once` primitive. +/// +/// This structure provides the Rust equivalent of the kernel's `DO_ONCE_LITE` macro. +/// While it would be possible to implement the feature entirely as a Rust macro, +/// the functionality that can be implemented as regular functions has been +/// extracted and implemented as the `OnceLite` struct for better code maintainability. +pub struct OnceLite(Atomic); + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +enum State { + Incomplete = 0, + Complete = 1, +} + +// SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip +// transmutable to `i32`. +unsafe impl AtomicType for State { + type Repr = i32; +} + +impl OnceLite { + /// Creates a new [`OnceLite`] in the incomplete state. + #[inline(always)] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + OnceLite(Atomic::new(State::Incomplete)) + } + + /// Calls the provided function exactly once. + /// + /// There is no other synchronization between two `call_once()`s + /// except that only one will execute `f`, in other words, callers + /// should not use a failed `call_once()` as a proof that another + /// `call_once()` has already finished and the effect is observable + /// to this thread. + pub fn call_once(&self, f: F) -> bool + where + F: FnOnce(), + { + // Avoid expensive cmpxchg if already completed. + // ORDERING: `Relaxed` is used here since no synchronization is required. + let old = self.0.load(Relaxed); + if old == State::Complete { + return false; + } + + // ORDERING: `Relaxed` is used here since no synchronization is required. + let old = self.0.xchg(State::Complete, Relaxed); + if old == State::Complete { + return false; + } + + f(); + true + } +} + +/// Run the given function exactly once. +/// +/// This is equivalent to the kernel's `DO_ONCE_LITE` macro. +/// +/// # Examples +/// +/// ``` +/// kernel::do_once_lite! { +/// kernel::pr_info!("This will be printed only once\n"); +/// }; +/// ``` +#[macro_export] +macro_rules! do_once_lite { + { $($e:tt)* } => {{ + #[link_section = ".data..once"] + static ONCE: $crate::print::OnceLite = $crate::print::OnceLite::new(); + ONCE.call_once(|| { $($e)* }); + }}; +} + +/// Prints an emergency-level message (level 0) only once. +/// +/// Equivalent to the kernel's `pr_emerg_once` macro. +#[macro_export] +macro_rules! pr_emerg_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_emerg!($($arg)*) } + ) +); + +/// Prints an alert-level message (level 1) only once. +/// +/// Equivalent to the kernel's `pr_alert_once` macro. +#[macro_export] +macro_rules! pr_alert_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_alert!($($arg)*) } + ) +); + +/// Prints a critical-level message (level 2) only once. +/// +/// Equivalent to the kernel's `pr_crit_once` macro. +#[macro_export] +macro_rules! pr_crit_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_crit!($($arg)*) } + ) +); + +/// Prints an error-level message (level 3) only once. +/// +/// Equivalent to the kernel's `pr_err_once` macro. +#[macro_export] +macro_rules! pr_err_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_err!($($arg)*) } + ) +); + +/// Prints a warning-level message (level 4) only once. +/// +/// Equivalent to the kernel's `pr_warn_once` macro. +#[macro_export] +macro_rules! pr_warn_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_warn!($($arg)*) } + ) +); + +/// Prints a notice-level message (level 5) only once. +/// +/// Equivalent to the kernel's `pr_notice_once` macro. +#[macro_export] +macro_rules! pr_notice_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_notice!($($arg)*) } + ) +); + +/// Prints an info-level message (level 6) only once. +/// +/// Equivalent to the kernel's `pr_info_once` macro. +#[macro_export] +macro_rules! pr_info_once ( + ($($arg:tt)*) => ( + $crate::do_once_lite! { $crate::pr_info!($($arg)*) } + ) +); diff --git a/rust/kernel/processor.rs b/rust/kernel/processor.rs new file mode 100644 index 000000000..85b49b361 --- /dev/null +++ b/rust/kernel/processor.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Processor related primitives. +//! +//! C header: [`include/linux/processor.h`](srctree/include/linux/processor.h) + +/// Lower CPU power consumption or yield to a hyperthreaded twin processor. +/// +/// It also happens to serve as a compiler barrier. +#[inline] +pub fn cpu_relax() { + // SAFETY: Always safe to call. + unsafe { bindings::cpu_relax() } +} diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs new file mode 100644 index 000000000..82acb531b --- /dev/null +++ b/rust/kernel/ptr.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types and functions to work with pointers and addresses. + +pub mod projection; +pub use crate::project_pointer as project; + +use core::mem::{ + align_of, + size_of, // +}; +use core::num::NonZero; + +use crate::const_assert; + +/// Type representing an alignment, which is always a power of two. +/// +/// It is used to validate that a given value is a valid alignment, and to perform masking and +/// alignment operations. +/// +/// This is a temporary substitute for the [`Alignment`] nightly type from the standard library, +/// and to be eventually replaced by it. +/// +/// [`Alignment`]: https://github.com/rust-lang/rust/issues/102070 +/// +/// # Invariants +/// +/// An alignment is always a power of two. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Alignment(NonZero); + +impl Alignment { + /// Validates that `ALIGN` is a power of two at build-time, and returns an [`Alignment`] of the + /// same value. + /// + /// A build error is triggered if `ALIGN` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// let v = Alignment::new::<16>(); + /// assert_eq!(v.as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn new() -> Self { + const_assert!( + ALIGN.is_power_of_two(), + "Provided alignment is not a power of two." + ); + + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Self(unsafe { NonZero::new_unchecked(ALIGN) }) + } + + /// Validates that `align` is a power of two at runtime, and returns an + /// [`Alignment`] of the same value. + /// + /// Returns [`None`] if `align` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new_checked(16), Some(Alignment::new::<16>())); + /// assert_eq!(Alignment::new_checked(15), None); + /// assert_eq!(Alignment::new_checked(1), Some(Alignment::new::<1>())); + /// assert_eq!(Alignment::new_checked(0), None); + /// ``` + #[inline(always)] + pub const fn new_checked(align: usize) -> Option { + if align.is_power_of_two() { + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Some(Self(unsafe { NonZero::new_unchecked(align) })) + } else { + None + } + } + + /// Returns the alignment of `T`. + /// + /// This is equivalent to [`align_of`], but with the return value provided as an [`Alignment`]. + #[inline(always)] + pub const fn of() -> Self { + // This cannot panic since alignments are always powers of two. + // + // We unfortunately cannot use `new` as it would require the `generic_const_exprs` feature. + const { Alignment::new_checked(align_of::()).unwrap() } + } + + /// Returns this alignment as a [`usize`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn as_usize(self) -> usize { + self.as_nonzero().get() + } + + /// Returns this alignment as a [`NonZero`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_nonzero().get(), 16); + /// ``` + #[inline(always)] + pub const fn as_nonzero(self) -> NonZero { + // Allow the compiler to know that the value is indeed a power of two. This can help + // optimize some operations down the line, like e.g. replacing divisions by bit shifts. + if !self.0.is_power_of_two() { + // SAFETY: Per the invariants, `self.0` is always a power of two so this block will + // never be reached. + unsafe { core::hint::unreachable_unchecked() } + } + self.0 + } + + /// Returns the base-2 logarithm of the alignment. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::of::().log2(), 0); + /// assert_eq!(Alignment::new::<16>().log2(), 4); + /// ``` + #[inline(always)] + pub const fn log2(self) -> u32 { + self.0.ilog2() + } + + /// Returns the mask for this alignment. + /// + /// This is equivalent to `!(self.as_usize() - 1)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<0x10>().mask(), !0xf); + /// ``` + #[inline(always)] + pub const fn mask(self) -> usize { + // No underflow can occur as the alignment is guaranteed to be a power of two, and thus is + // non-zero. + !(self.as_usize() - 1) + } +} + +/// Trait for items that can be aligned against an [`Alignment`]. +pub trait Alignable: Sized { + /// Aligns `self` down to `alignment`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x2f_usize.align_down(Alignment::new::<0x10>()), 0x20); + /// assert_eq!(0x30usize.align_down(Alignment::new::<0x10>()), 0x30); + /// assert_eq!(0xf0u8.align_down(Alignment::new::<0x1000>()), 0x0); + /// ``` + fn align_down(self, alignment: Alignment) -> Self; + + /// Aligns `self` up to `alignment`, returning `None` if aligning would result in an overflow. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x4fusize.align_up(Alignment::new::<0x10>()), Some(0x50)); + /// assert_eq!(0x40usize.align_up(Alignment::new::<0x10>()), Some(0x40)); + /// assert_eq!(0x0usize.align_up(Alignment::new::<0x10>()), Some(0x0)); + /// assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None); + /// assert_eq!(0x10u8.align_up(Alignment::new::<0x100>()), None); + /// assert_eq!(0x0u8.align_up(Alignment::new::<0x100>()), Some(0x0)); + /// ``` + fn align_up(self, alignment: Alignment) -> Option; +} + +/// Implement [`Alignable`] for unsigned integer types. +macro_rules! impl_alignable_uint { + ($($t:ty),*) => { + $( + impl Alignable for $t { + #[inline(always)] + fn align_down(self, alignment: Alignment) -> Self { + // The operands of `&` need to be of the same type so convert the alignment to + // `Self`. This means we need to compute the mask ourselves. + ::core::num::NonZero::::try_from(alignment.as_nonzero()) + .map(|align| self & !(align.get() - 1)) + // An alignment larger than `Self` always aligns down to `0`. + .unwrap_or(0) + } + + #[inline(always)] + fn align_up(self, alignment: Alignment) -> Option { + let aligned_down = self.align_down(alignment); + if self == aligned_down { + Some(aligned_down) + } else { + Self::try_from(alignment.as_usize()) + .ok() + .and_then(|align| aligned_down.checked_add(align)) + } + } + } + )* + }; +} + +impl_alignable_uint!(u8, u16, u32, u64, usize); + +/// Trait to represent compile-time known size information. +/// +/// This is a generalization of [`size_of`] that works for dynamically sized types. +pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. + fn size(p: *const Self) -> usize; +} + +impl KnownSize for T { + const MIN_SIZE: usize = size_of::(); + const MIN_ALIGN: Alignment = Alignment::of::(); + + #[inline(always)] + fn size(_: *const Self) -> usize { + size_of::() + } +} + +impl KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + p.len() * size_of::() + } +} + +/// Aligns `value` up to `align`. +/// +/// This is the const-compatible equivalent of [`Alignable::align_up`]. +/// +/// Returns [`None`] on overflow. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// ptr::{ +/// const_align_up, +/// Alignment, // +/// }, +/// sizes::SZ_4K, // +/// }; +/// +/// assert_eq!(const_align_up(0x4f, Alignment::new::<16>()), Some(0x50)); +/// assert_eq!(const_align_up(0x40, Alignment::new::<16>()), Some(0x40)); +/// assert_eq!(const_align_up(1, Alignment::new::()), Some(SZ_4K)); +/// ``` +#[inline(always)] +pub const fn const_align_up(value: usize, align: Alignment) -> Option { + match value.checked_add(align.as_usize() - 1) { + Some(v) => Some(v & align.mask()), + None => None, + } +} diff --git a/rust/kernel/ptr/projection.rs b/rust/kernel/ptr/projection.rs new file mode 100644 index 000000000..af72d3b0e --- /dev/null +++ b/rust/kernel/ptr/projection.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for handling projections. + +use core::{ + mem::MaybeUninit, + ops::Deref, // +}; + +use crate::prelude::*; + +/// Error raised when a projection is attempted on an array or slice out of bounds. +pub struct OutOfBound; + +impl From for Error { + #[inline(always)] + fn from(_: OutOfBound) -> Self { + ERANGE + } +} + +/// A helper trait to perform index projection. +/// +/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and +/// fallibly. +/// +/// # Safety +/// +/// For a given input pointer `slice` and return value `output`, the implementation of `index`, +/// `build_index` and `get` (if [`Some`] is returned) must ensure that: +/// - `output` has the same provenance as `slice`; +/// - `output.byte_offset_from(slice)` is between 0 to +/// `KnownSize::size(slice) - KnownSize::size(output)`. +/// +/// This means that if the input pointer is valid, then the pointer returned by `get`, `index` +/// or `build_index` is also valid. +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] +#[doc(hidden)] +pub unsafe trait ProjectIndex: Sized { + type Output: ?Sized; + + /// Returns an index-projected pointer, if in bounds. + fn get(self, slice: *mut T) -> Option<*mut Self::Output>; + + /// Returns an index-projected pointer; panic if out of bounds. + fn index(self, slice: *mut T) -> *mut Self::Output; + + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. + #[inline(always)] + fn build_index(self, slice: *mut T) -> *mut Self::Output { + match Self::get(self, slice) { + Some(v) => v, + None => build_error!(), + } + } +} + +// Forward array impl to slice impl. +// +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T; N]> for I +where + I: ProjectIndex<[T]>, +{ + type Output = >::Output; + + #[inline(always)] + fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> { + >::get(self, slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T; N]) -> *mut Self::Output { + >::index(self, slice) + } + + #[inline(always)] + fn build_index(self, slice: *mut [T; N]) -> *mut Self::Output { + >::build_index(self, slice) + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl ProjectIndex<[T]> for usize { + type Output = T; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut T> { + if self >= slice.len() { + None + } else { + Some(slice.cast::().wrapping_add(self)) + } + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut T { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + let () = zst_slice[self]; + slice.cast::().wrapping_add(self) + } +} + +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to +// not exceed the required bound. +unsafe impl ProjectIndex<[T]> for core::ops::Range { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + let new_len = self.end.checked_sub(self.start)?; + if self.end > slice.len() { + return None; + } + Some(core::ptr::slice_from_raw_parts_mut( + slice.cast::().wrapping_add(self.start), + new_len, + )) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + _ = zst_slice[self.clone()]; + + // SAFETY: Bounds checked. + unsafe { self.get(slice).unwrap_unchecked() } + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T]> for core::ops::RangeTo { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (0..self.end).get(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (0..self.end).index(slice) + } +} + +// SAFETY: Safety requirement guaranteed by the forwarded impl. +unsafe impl ProjectIndex<[T]> for core::ops::RangeFrom { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + (self.start..slice.len()).get(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (self.start..slice.len()).index(slice) + } +} + +// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. +unsafe impl ProjectIndex<[T]> for core::ops::RangeFull { + type Output = [T]; + + #[inline(always)] + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { + Some(slice) + } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + slice + } +} + +/// A helper trait to perform field projection. +/// +/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that +/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being +/// used as base of projection, as they can inject unsoundness. Users therefore must not specify +/// `DEREF` and should always leave it to be inferred. +/// +/// # Safety +/// +/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`] +/// describes. +#[doc(hidden)] +pub unsafe trait ProjectField { + /// Project a pointer to a type to a pointer of a field. + /// + /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to + /// fields using `&raw mut`. + /// + /// This is needed because `base` might not point to a valid allocation, while `&raw mut` + /// requires pointers to be in bounds of a valid allocation. + /// + /// # Safety + /// + /// `f` must return a pointer in bounds of the provided pointer. + unsafe fn proj(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F; +} + +// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot +// currently support that as we need to obtain a valid allocation that `&raw const` can operate on. +// +// SAFETY: `proj` invokes `f` with valid allocation. +unsafe impl ProjectField for T { + #[inline(always)] + unsafe fn proj(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + // Create a valid allocation to start projection, as `base` is not necessarily so. The + // memory is never actually used so it will be optimized out, so it should work even for + // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also + // annotate `f` closure with `#[inline(always)]` in the macro. + let mut place = MaybeUninit::uninit(); + let place_base = place.as_mut_ptr(); + let field = f(place_base); + // SAFETY: `field` is in bounds from `base` per safety requirement. + let offset = unsafe { field.byte_offset_from(place_base) }; + // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation. + base.wrapping_byte_offset(offset).cast() + } +} + +// SAFETY: Vacuously satisfied. +unsafe impl ProjectField for T { + #[inline(always)] + unsafe fn proj(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F { + build_error!("this function is a guard against `Deref` impl and is never invoked"); + } +} + +/// Create a projection from a raw pointer. +/// +/// The projected pointer is within the memory region marked by the input pointer. There is no +/// requirement that the input raw pointer needs to be valid, so this macro may be used for +/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input +/// pointer is valid, the projected pointer is also valid. +/// +/// Supported projections include field projections and index projections. +/// It is not allowed to project into types that implement custom [`Deref`] or +/// [`Index`](core::ops::Index). +/// +/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an +/// expression that evaluates to a raw pointer which serves as the base of projection. `projection` +/// can be a projection expression of form `.field` (normally identifier, or numeral in case of +/// tuple structs) or of form `[index]`. +/// +/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. +/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. +/// +/// The `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. +/// The syntax is of the form `[: index]` where `flavor` indicates the way of handling +/// index out-of-bounds errors. +/// - `try` will raise an [`OutOfBound`] error (which is convertible to [`ERANGE`]). +/// - `build` will use the [`build_assert!`] mechanism to have the compiler validate the index is +/// in bounds. +/// - `panic` will cause a Rust [`panic!`] if the index goes out of bounds. +/// +/// # Examples +/// +/// Field projections are performed with `.field_name`: +/// +/// ``` +/// struct MyStruct { field: u32, } +/// let ptr: *const MyStruct = core::ptr::dangling(); +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field); +/// +/// struct MyTupleStruct(u32, u32); +/// +/// fn proj(ptr: *const MyTupleStruct) { +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1); +/// } +/// ``` +/// +/// Index projections are performed with `[: index]`, where `flavor` is `try`, `build` or +/// `panic`: +/// +/// ``` +/// fn proj(ptr: *const [u8; 32]) -> Result { +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [build: 1]); +/// // The following invocation, if uncommented, would fail the build. +/// // +/// // kernel::ptr::project!(ptr, [build: 128]); +/// +/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). +/// kernel::ptr::project!(ptr, [try: 128]); +/// +/// // This will panic at runtime if executed. +/// kernel::ptr::project!(ptr, [panic: 128]); +/// Ok(()) +/// } +/// ``` +/// +/// If you need to match on the error instead of propagate, put the invocation inside a closure: +/// +/// ``` +/// let ptr: *const [u8; 32] = core::ptr::dangling(); +/// let field_ptr: Result<*const u8> = (|| -> Result<_> { +/// Ok(kernel::ptr::project!(ptr, [try: 128])) +/// })(); +/// assert!(field_ptr.is_err()); +/// ``` +/// +/// For mutable pointers, put `mut` as the first token in macro invocation. +/// +/// ``` +/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [build: 1].1); +/// ``` +#[macro_export] +macro_rules! project_pointer { + (@gen $ptr:ident, ) => {}; + // Field projection. `$field` needs to be `tt` to support tuple index like `.0`. + (@gen $ptr:ident, .$field:tt $($rest:tt)*) => { + // SAFETY: The provided closure always returns an in-bounds pointer. + let $ptr = unsafe { + $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| { + // Check unaligned field. Not all users (e.g. DMA) can handle unaligned + // projections. + if false { + let _ = &(*ptr).$field; + } + // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the + // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`). + &raw mut (*ptr).$field + }) + }; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Fallible index projection. + (@gen $ptr:ident, [try: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) + .ok_or($crate::ptr::projection::OutOfBound)?; + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Panicking index projection. + (@gen $ptr:ident, [panic: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + // Build-time checked index projection. + (@gen $ptr:ident, [build: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::build_index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + + (mut $ptr:expr, $($proj:tt)*) => {{ + let ptr: *mut _ = $ptr; + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr + }}; + ($ptr:expr, $($proj:tt)*) => {{ + let ptr = <*const _>::cast_mut($ptr); + // We currently always project using mutable pointer, as it is not decided whether `&raw + // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`). + $crate::ptr::project!(@gen ptr, $($proj)*); + ptr.cast_const() + }}; +} diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs new file mode 100644 index 000000000..8aa47304b --- /dev/null +++ b/rust/kernel/pwm.rs @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2025 Samsung Electronics Co., Ltd. +// Author: Michal Wilczynski + +//! PWM subsystem abstractions. +//! +//! C header: [`include/linux/pwm.h`](srctree/include/linux/pwm.h). + +use crate::{ + bindings, + container_of, + device::{self, Bound}, + devres, + error::{self, to_result}, + prelude::*, + sync::aref::{ARef, AlwaysRefCounted}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::NonNull, // +}; + +/// Represents a PWM waveform configuration. +/// Mirrors struct [`struct pwm_waveform`](srctree/include/linux/pwm.h). +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct Waveform { + /// Total duration of one complete PWM cycle, in nanoseconds. + pub period_length_ns: u64, + + /// Duty-cycle active time, in nanoseconds. + /// + /// For a typical normal polarity configuration (active-high) this is the + /// high time of the signal. + pub duty_length_ns: u64, + + /// Duty-cycle start offset, in nanoseconds. + /// + /// Delay from the beginning of the period to the first active edge. + /// In most simple PWM setups this is `0`, so the duty cycle starts + /// immediately at each period’s start. + pub duty_offset_ns: u64, +} + +impl From for Waveform { + fn from(wf: bindings::pwm_waveform) -> Self { + Waveform { + period_length_ns: wf.period_length_ns, + duty_length_ns: wf.duty_length_ns, + duty_offset_ns: wf.duty_offset_ns, + } + } +} + +impl From for bindings::pwm_waveform { + fn from(wf: Waveform) -> Self { + bindings::pwm_waveform { + period_length_ns: wf.period_length_ns, + duty_length_ns: wf.duty_length_ns, + duty_offset_ns: wf.duty_offset_ns, + } + } +} + +/// Describes the outcome of a `round_waveform` operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoundingOutcome { + /// The requested waveform was achievable exactly or by rounding values down. + ExactOrRoundedDown, + + /// The requested waveform could only be achieved by rounding up. + RoundedUp, +} + +/// Wrapper for a PWM device [`struct pwm_device`](srctree/include/linux/pwm.h). +#[repr(transparent)] +pub struct Device(Opaque); + +impl Device { + /// Creates a reference to a [`Device`] from a valid C pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Device`] reference. + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::pwm_device) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Device` type being transparent makes the cast ok. + unsafe { &*ptr.cast::() } + } + + /// Returns a raw pointer to the underlying `pwm_device`. + fn as_raw(&self) -> *mut bindings::pwm_device { + self.0.get() + } + + /// Gets the hardware PWM index for this device within its chip. + pub fn hwpwm(&self) -> u32 { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).hwpwm } + } + + /// Gets a reference to the parent `Chip` that this device belongs to. + pub fn chip(&self) -> &Chip { + // SAFETY: `self.as_raw()` provides a valid pointer. (*self.as_raw()).chip + // is assumed to be a valid pointer to `pwm_chip` managed by the kernel. + // Chip::from_raw's safety conditions must be met. + unsafe { Chip::::from_raw((*self.as_raw()).chip) } + } + + /// Gets the label for this PWM device, if any. + pub fn label(&self) -> Option<&CStr> { + // SAFETY: self.as_raw() provides a valid pointer. + let label_ptr = unsafe { (*self.as_raw()).label }; + if label_ptr.is_null() { + return None; + } + + // SAFETY: label_ptr is non-null and points to a C string + // managed by the kernel, valid for the lifetime of the PWM device. + Some(unsafe { CStr::from_char_ptr(label_ptr) }) + } + + /// Sets the PWM waveform configuration and enables the PWM signal. + pub fn set_waveform(&self, wf: &Waveform, exact: bool) -> Result { + let c_wf = bindings::pwm_waveform::from(*wf); + + // SAFETY: `self.as_raw()` provides a valid `*mut pwm_device` pointer. + // `&c_wf` is a valid pointer to a `pwm_waveform` struct. The C function + // handles all necessary internal locking. + to_result(unsafe { bindings::pwm_set_waveform_might_sleep(self.as_raw(), &c_wf, exact) }) + } + + /// Queries the hardware for the configuration it would apply for a given + /// request. + pub fn round_waveform(&self, wf: &mut Waveform) -> Result { + let mut c_wf = bindings::pwm_waveform::from(*wf); + + // SAFETY: `self.as_raw()` provides a valid `*mut pwm_device` pointer. + // `&mut c_wf` is a valid pointer to a mutable `pwm_waveform` struct that + // the C function will update. + let ret = unsafe { bindings::pwm_round_waveform_might_sleep(self.as_raw(), &mut c_wf) }; + + to_result(ret)?; + + *wf = Waveform::from(c_wf); + + if ret == 1 { + Ok(RoundingOutcome::RoundedUp) + } else { + Ok(RoundingOutcome::ExactOrRoundedDown) + } + } + + /// Reads the current waveform configuration directly from the hardware. + pub fn get_waveform(&self) -> Result { + let mut c_wf = bindings::pwm_waveform::default(); + + // SAFETY: `self.as_raw()` is a valid pointer. We provide a valid pointer + // to a stack-allocated `pwm_waveform` struct for the kernel to fill. + to_result(unsafe { bindings::pwm_get_waveform_might_sleep(self.as_raw(), &mut c_wf) })?; + + Ok(Waveform::from(c_wf)) + } +} + +/// The result of a `round_waveform_tohw` operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RoundedWaveform { + /// A status code, 0 for success or 1 if values were rounded up. + pub status: c_int, + /// The driver-specific hardware representation of the waveform. + pub hardware_waveform: WfHw, +} + +/// Trait defining the operations for a PWM driver. +pub trait PwmOps: 'static + Send + Sync + Sized { + /// The driver-specific hardware representation of a waveform. + /// + /// This type must be [`Copy`], [`Default`], and fit within `PWM_WFHWSIZE`. + type WfHw: Copy + Default; + + /// Optional hook for when a PWM device is requested. + fn request(_chip: &Chip, _pwm: &Device, _parent_dev: &device::Device) -> Result { + Ok(()) + } + + /// Optional hook for capturing a PWM signal. + fn capture( + _chip: &Chip, + _pwm: &Device, + _result: &mut bindings::pwm_capture, + _timeout: usize, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } + + /// Convert a generic waveform to the hardware-specific representation. + /// This is typically a pure calculation and does not perform I/O. + fn round_waveform_tohw( + _chip: &Chip, + _pwm: &Device, + _wf: &Waveform, + ) -> Result> { + Err(ENOTSUPP) + } + + /// Convert a hardware-specific representation back to a generic waveform. + /// This is typically a pure calculation and does not perform I/O. + fn round_waveform_fromhw( + _chip: &Chip, + _pwm: &Device, + _wfhw: &Self::WfHw, + _wf: &mut Waveform, + ) -> Result { + Err(ENOTSUPP) + } + + /// Read the current hardware configuration into the hardware-specific representation. + fn read_waveform( + _chip: &Chip, + _pwm: &Device, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } + + /// Write a hardware-specific waveform configuration to the hardware. + fn write_waveform( + _chip: &Chip, + _pwm: &Device, + _wfhw: &Self::WfHw, + _parent_dev: &device::Device, + ) -> Result { + Err(ENOTSUPP) + } +} + +/// Bridges Rust `PwmOps` to the C `pwm_ops` vtable. +struct Adapter { + _p: PhantomData, +} + +impl Adapter { + const VTABLE: PwmOpsVTable = create_pwm_ops::(); + + /// # Safety + /// + /// `wfhw_ptr` must be valid for writes of `size_of::()` bytes. + unsafe fn serialize_wfhw(wfhw: &T::WfHw, wfhw_ptr: *mut c_void) -> Result { + let size = core::mem::size_of::(); + + build_assert!(size <= bindings::PWM_WFHWSIZE as usize); + + // SAFETY: The caller ensures `wfhw_ptr` is valid for `size` bytes. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref::(wfhw).cast::(), + wfhw_ptr.cast::(), + size, + ) + }; + + Ok(()) + } + + /// # Safety + /// + /// `wfhw_ptr` must be valid for reads of `size_of::()` bytes. + unsafe fn deserialize_wfhw(wfhw_ptr: *const c_void) -> Result { + let size = core::mem::size_of::(); + + build_assert!(size <= bindings::PWM_WFHWSIZE as usize); + + let mut wfhw = T::WfHw::default(); + // SAFETY: The caller ensures `wfhw_ptr` is valid for `size` bytes. + unsafe { + core::ptr::copy_nonoverlapping( + wfhw_ptr.cast::(), + core::ptr::from_mut::(&mut wfhw).cast::(), + size, + ) + }; + + Ok(wfhw) + } + + /// # Safety + /// + /// `dev` must be a valid pointer to a `bindings::device` embedded within a + /// `bindings::pwm_chip`. This function is called by the device core when the + /// last reference to the device is dropped. + unsafe extern "C" fn release_callback(dev: *mut bindings::device) { + // SAFETY: The function's contract guarantees that `dev` points to a `device` + // field embedded within a valid `pwm_chip`. `container_of!` can therefore + // safely calculate the address of the containing struct. + let c_chip_ptr = unsafe { container_of!(dev, bindings::pwm_chip, dev) }; + + // SAFETY: `c_chip_ptr` is a valid pointer to a `pwm_chip` as established + // above. Calling this FFI function is safe. + let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; + + // SAFETY: The driver data was initialized in `new`. We run its destructor here. + unsafe { core::ptr::drop_in_place(drvdata_ptr.cast::()) }; + + // Now, call the original release function to free the `pwm_chip` itself. + // SAFETY: `dev` is the valid pointer passed into this callback, which is + // the expected argument for `pwmchip_release`. + unsafe { bindings::pwmchip_release(dev) }; + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn request_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + ) -> c_int { + // SAFETY: PWM core guarentees `chip_ptr` and `pwm_ptr` are valid pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::request(chip, pwm, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn capture_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + res: *mut bindings::pwm_capture, + timeout: usize, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm, result) = unsafe { + ( + Chip::::from_raw(chip_ptr), + Device::from_raw(pwm_ptr), + &mut *res, + ) + }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::capture(chip, pwm, result, timeout, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn round_waveform_tohw_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wf_ptr: *const bindings::pwm_waveform, + wfhw_ptr: *mut c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm, wf) = unsafe { + ( + Chip::::from_raw(chip_ptr), + Device::from_raw(pwm_ptr), + Waveform::from(*wf_ptr), + ) + }; + match T::round_waveform_tohw(chip, pwm, &wf) { + Ok(rounded) => { + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + if unsafe { Self::serialize_wfhw(&rounded.hardware_waveform, wfhw_ptr) }.is_err() { + return EINVAL.to_errno(); + } + rounded.status + } + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn round_waveform_fromhw_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *const c_void, + wf_ptr: *mut bindings::pwm_waveform, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + // SAFETY: `deserialize_wfhw`'s safety contract is met by this function's contract. + let wfhw = match unsafe { Self::deserialize_wfhw(wfhw_ptr) } { + Ok(v) => v, + Err(e) => return e.to_errno(), + }; + + let mut rust_wf = Waveform::default(); + match T::round_waveform_fromhw(chip, pwm, &wfhw, &mut rust_wf) { + Ok(()) => { + // SAFETY: `wf_ptr` is guaranteed valid by the C caller. + unsafe { *wf_ptr = rust_wf.into() }; + 0 + } + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn read_waveform_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *mut c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + match T::read_waveform(chip, pwm, bound_parent) { + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + Ok(wfhw) => match unsafe { Self::serialize_wfhw(&wfhw, wfhw_ptr) } { + Ok(()) => 0, + Err(e) => e.to_errno(), + }, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// Pointers from C must be valid. + unsafe extern "C" fn write_waveform_callback( + chip_ptr: *mut bindings::pwm_chip, + pwm_ptr: *mut bindings::pwm_device, + wfhw_ptr: *const c_void, + ) -> c_int { + // SAFETY: Relies on the function's contract that `chip_ptr` and `pwm_ptr` are valid + // pointers. + let (chip, pwm) = unsafe { (Chip::::from_raw(chip_ptr), Device::from_raw(pwm_ptr)) }; + + // SAFETY: The PWM core guarantees the parent device exists and is bound during callbacks. + let bound_parent = unsafe { chip.bound_parent_device() }; + + // SAFETY: `wfhw_ptr` is valid per this function's safety contract. + let wfhw = match unsafe { Self::deserialize_wfhw(wfhw_ptr) } { + Ok(v) => v, + Err(e) => return e.to_errno(), + }; + match T::write_waveform(chip, pwm, &wfhw, bound_parent) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + } +} + +/// VTable structure wrapper for PWM operations. +/// Mirrors [`struct pwm_ops`](srctree/include/linux/pwm.h). +#[repr(transparent)] +pub struct PwmOpsVTable(bindings::pwm_ops); + +// SAFETY: PwmOpsVTable is Send. The vtable contains only function pointers +// and a size, which are simple data types that can be safely moved across +// threads. The thread-safety of calling these functions is handled by the +// kernel's locking mechanisms. +unsafe impl Send for PwmOpsVTable {} + +// SAFETY: PwmOpsVTable is Sync. The vtable is immutable after it is created, +// so it can be safely referenced and accessed concurrently by multiple threads +// e.g. to read the function pointers. +unsafe impl Sync for PwmOpsVTable {} + +impl PwmOpsVTable { + /// Returns a raw pointer to the underlying `pwm_ops` struct. + pub(crate) fn as_raw(&self) -> *const bindings::pwm_ops { + &self.0 + } +} + +/// Creates a PWM operations vtable for a type `T` that implements `PwmOps`. +/// +/// This is used to bridge Rust trait implementations to the C `struct pwm_ops` +/// expected by the kernel. +pub const fn create_pwm_ops() -> PwmOpsVTable { + let mut ops: bindings::pwm_ops = pin_init::zeroed(); + + ops.request = Some(Adapter::::request_callback); + ops.capture = Some(Adapter::::capture_callback); + + ops.round_waveform_tohw = Some(Adapter::::round_waveform_tohw_callback); + ops.round_waveform_fromhw = Some(Adapter::::round_waveform_fromhw_callback); + ops.read_waveform = Some(Adapter::::read_waveform_callback); + ops.write_waveform = Some(Adapter::::write_waveform_callback); + ops.sizeof_wfhw = core::mem::size_of::(); + + PwmOpsVTable(ops) +} + +/// Wrapper for a PWM chip/controller ([`struct pwm_chip`](srctree/include/linux/pwm.h)). +#[repr(transparent)] +pub struct Chip(Opaque, PhantomData); + +impl Chip { + /// Creates a reference to a [`Chip`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`Chip`] reference. + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::pwm_chip) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `Chip` type being transparent makes the cast ok. + unsafe { &*ptr.cast::() } + } + + /// Returns a raw pointer to the underlying `pwm_chip`. + pub(crate) fn as_raw(&self) -> *mut bindings::pwm_chip { + self.0.get() + } + + /// Gets the number of PWM channels (hardware PWMs) on this chip. + pub fn num_channels(&self) -> u32 { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).npwm } + } + + /// Returns `true` if the chip supports atomic operations for configuration. + pub fn is_atomic(&self) -> bool { + // SAFETY: `self.as_raw()` provides a valid pointer for `self`'s lifetime. + unsafe { (*self.as_raw()).atomic } + } + + /// Returns a reference to the embedded `struct device` abstraction. + pub fn device(&self) -> &device::Device { + // SAFETY: + // - `self.as_raw()` provides a valid pointer to `bindings::pwm_chip`. + // - The `dev` field is an instance of `bindings::device` embedded + // within `pwm_chip`. + // - Taking a pointer to this embedded field is valid. + // - `device::Device` is `#[repr(transparent)]`. + // - The lifetime of the returned reference is tied to `self`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } + + /// Gets the typed driver specific data associated with this chip's embedded device. + pub fn drvdata(&self) -> &T { + // SAFETY: `pwmchip_get_drvdata` returns the pointer to the private data area, + // which we know holds our `T`. The pointer is valid for the lifetime of `self`. + unsafe { &*bindings::pwmchip_get_drvdata(self.as_raw()).cast::() } + } + + /// Returns a reference to the parent device of this PWM chip's device. + /// + /// # Safety + /// + /// The caller must guarantee that the parent device exists and is bound. + /// This is guaranteed by the PWM core during `PwmOps` callbacks. + unsafe fn bound_parent_device(&self) -> &device::Device { + // SAFETY: Per the function's safety contract, the parent device exists. + let parent = unsafe { self.device().parent().unwrap_unchecked() }; + + // SAFETY: Per the function's safety contract, the parent device is bound. + // This is guaranteed by the PWM core during `PwmOps` callbacks. + unsafe { parent.as_bound() } + } + + /// Allocates and wraps a PWM chip using `bindings::pwmchip_alloc`. + /// + /// Returns an [`ARef`] managing the chip's lifetime via refcounting + /// on its embedded `struct device`. + #[allow(clippy::new_ret_no_self)] + pub fn new<'a>( + parent_dev: &'a device::Device, + num_channels: u32, + data: impl pin_init::PinInit, + ) -> Result> { + let sizeof_priv = core::mem::size_of::(); + // SAFETY: `pwmchip_alloc` allocates memory for the C struct and our private data. + let c_chip_ptr_raw = + unsafe { bindings::pwmchip_alloc(parent_dev.as_raw(), num_channels, sizeof_priv) }; + + let c_chip_ptr: *mut bindings::pwm_chip = error::from_err_ptr(c_chip_ptr_raw)?; + + // SAFETY: The `drvdata` pointer is the start of the private area, which is where + // we will construct our `T` object. + let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; + + // SAFETY: We construct the `T` object in-place in the allocated private memory. + unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| { + // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained + // from `pwmchip_alloc()`. We will not use pointer after this. + unsafe { bindings::pwmchip_put(c_chip_ptr) } + })?; + + // SAFETY: `c_chip_ptr` points to a valid chip. + unsafe { (*c_chip_ptr).dev.release = Some(Adapter::::release_callback) }; + + // SAFETY: `c_chip_ptr` points to a valid chip. + // The `Adapter`'s `VTABLE` has a 'static lifetime, so the pointer + // returned by `as_raw()` is always valid. + unsafe { (*c_chip_ptr).ops = Adapter::::VTABLE.as_raw() }; + + // Cast the `*mut bindings::pwm_chip` to `*mut Chip`. This is valid because + // `Chip` is `repr(transparent)` over `Opaque`, and + // `Opaque` is `repr(transparent)` over `T`. + let chip_ptr_as_self = c_chip_ptr.cast::(); + + // SAFETY: `chip_ptr_as_self` points to a valid `Chip` (layout-compatible with + // `bindings::pwm_chip`) whose embedded device has refcount 1. + // `ARef::from_raw` takes this pointer and manages it via `AlwaysRefCounted`. + let chip = unsafe { ARef::from_raw(NonNull::new_unchecked(chip_ptr_as_self)) }; + + Ok(UnregisteredChip { chip, parent_dev }) + } +} + +// SAFETY: Implements refcounting for `Chip` using the embedded `struct device`. +unsafe impl AlwaysRefCounted for Chip { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self.0.get()` points to a valid `pwm_chip` because `self` exists. + // The embedded `dev` is valid. `get_device` increments its refcount. + unsafe { bindings::get_device(&raw mut (*self.0.get()).dev) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull>) { + let c_chip_ptr = obj.cast::().as_ptr(); + + // SAFETY: `obj` is a valid pointer to a `Chip` (and thus `bindings::pwm_chip`) + // with a non-zero refcount. `put_device` handles decrement and final release. + unsafe { bindings::put_device(&raw mut (*c_chip_ptr).dev) }; + } +} + +// SAFETY: `Chip` is a wrapper around `*mut bindings::pwm_chip`. The underlying C +// structure's state is managed and synchronized by the kernel's device model +// and PWM core locking mechanisms. Therefore, it is safe to move the `Chip` +// wrapper (and the pointer it contains) across threads. +unsafe impl Send for Chip {} + +// SAFETY: It is safe for multiple threads to have shared access (`&Chip`) because +// the `Chip` data is immutable from the Rust side without holding the appropriate +// kernel locks, which the C core is responsible for. Any interior mutability is +// handled and synchronized by the C kernel code. +unsafe impl Sync for Chip {} + +/// A wrapper around `ARef>` that ensures that `register` can only be called once. +pub struct UnregisteredChip<'a, T: PwmOps> { + chip: ARef>, + parent_dev: &'a device::Device, +} + +impl UnregisteredChip<'_, T> { + /// Registers a PWM chip with the PWM subsystem. + /// + /// Transfers its ownership to the `devres` framework, which ties its lifetime + /// to the parent device. + /// On unbind of the parent device, the `devres` entry will be dropped, automatically + /// calling `pwmchip_remove`. This function should be called from the driver's `probe`. + pub fn register(self) -> Result>> { + let c_chip_ptr = self.chip.as_raw(); + + // SAFETY: `c_chip_ptr` points to a valid chip with its ops initialized. + // `__pwmchip_add` is the C function to register the chip with the PWM core. + to_result(unsafe { bindings::__pwmchip_add(c_chip_ptr, core::ptr::null_mut()) })?; + + let registration = Registration { + chip: ARef::clone(&self.chip), + }; + + devres::register(self.parent_dev, registration, GFP_KERNEL)?; + + Ok(self.chip) + } +} + +impl Deref for UnregisteredChip<'_, T> { + type Target = Chip; + + fn deref(&self) -> &Self::Target { + &self.chip + } +} + +/// A resource guard that ensures `pwmchip_remove` is called on drop. +/// +/// This struct is intended to be managed by the `devres` framework by transferring its ownership +/// via [`devres::register`]. This ties the lifetime of the PWM chip registration +/// to the lifetime of the underlying device. +struct Registration { + chip: ARef>, +} + +impl Drop for Registration { + fn drop(&mut self) { + let chip_raw = self.chip.as_raw(); + + // SAFETY: `chip_raw` points to a chip that was successfully registered. + // `bindings::pwmchip_remove` is the correct C function to unregister it. + // This `drop` implementation is called automatically by `devres` on driver unbind. + unsafe { bindings::pwmchip_remove(chip_raw) }; + } +} + +/// Declares a kernel module that exposes a single PWM driver. +/// +/// # Examples +/// +///```ignore +/// kernel::module_pwm_platform_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +///``` +#[macro_export] +macro_rules! module_pwm_platform_driver { + ($($user_args:tt)*) => { + $crate::module_platform_driver! { + $($user_args)* + imports_ns: ["PWM"], + } + }; +} diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs new file mode 100644 index 000000000..6fbd579d4 --- /dev/null +++ b/rust/kernel/rbtree.rs @@ -0,0 +1,1433 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Red-black trees. +//! +//! C header: [`include/linux/rbtree.h`](srctree/include/linux/rbtree.h) +//! +//! Reference: + +use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; +use core::{ + cmp::{Ord, Ordering}, + marker::PhantomData, + mem::MaybeUninit, + ptr::{addr_of_mut, from_mut, NonNull}, +}; + +/// A red-black tree with owned nodes. +/// +/// It is backed by the kernel C red-black trees. +/// +/// # Examples +/// +/// In the example below we do several operations on a tree. We note that insertions may fail if +/// the system is out of memory. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// assert_eq!(tree.get(&10), Some(&100)); +/// assert_eq!(tree.get(&20), Some(&200)); +/// assert_eq!(tree.get(&30), Some(&300)); +/// } +/// +/// // Iterate over the nodes we just inserted. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Print all elements. +/// for (key, value) in &tree { +/// pr_info!("{} = {}\n", key, value); +/// } +/// +/// // Replace one of the elements. +/// tree.try_create_and_insert(10, 1000, flags::GFP_KERNEL)?; +/// +/// // Check that the tree reflects the replacement. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Change the value of one of the elements. +/// *tree.get_mut(&30).unwrap() = 3000; +/// +/// // Check that the tree reflects the update. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Remove an element. +/// tree.remove(&10); +/// +/// // Check that the tree reflects the removal. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); +/// assert!(iter.next().is_none()); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// In the example below, we first allocate a node, acquire a spinlock, then insert the node into +/// the tree. This is useful when the insertion context does not allow sleeping, for example, when +/// holding a spinlock. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode}, sync::SpinLock}; +/// +/// fn insert_test(tree: &SpinLock>) -> Result { +/// // Pre-allocate node. This may fail (as it allocates memory). +/// let node = RBTreeNode::new(10, 100, flags::GFP_KERNEL)?; +/// +/// // Insert node while holding the lock. It is guaranteed to succeed with no allocation +/// // attempts. +/// let mut guard = tree.lock(); +/// guard.insert(node); +/// Ok(()) +/// } +/// ``` +/// +/// In the example below, we reuse an existing node allocation from an element we removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Remove a node, getting back ownership of it. +/// let existing = tree.remove(&30); +/// +/// // Check that the tree reflects the removal. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Create a preallocated reservation that we can re-use later. +/// let reservation = RBTreeNodeReservation::new(flags::GFP_KERNEL)?; +/// +/// // Insert a new node into the tree, reusing the previous allocation. This is guaranteed to +/// // succeed (no memory allocations). +/// tree.insert(reservation.into_node(15, 150)); +/// +/// // Check that the tree reflect the new insertion. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&15, &150))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert!(iter.next().is_none()); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// Non-null parent/children pointers stored in instances of the `rb_node` C struct are always +/// valid, and pointing to a field of our internal representation of a node. +pub struct RBTree { + root: bindings::rb_root, + _p: PhantomData>, +} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Send condition as would be used for a struct with K and V fields. +unsafe impl Send for RBTree {} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Sync condition as would be used for a struct with K and V fields. +unsafe impl Sync for RBTree {} + +impl RBTree { + /// Creates a new and empty tree. + pub fn new() -> Self { + Self { + // INVARIANT: There are no nodes in the tree, so the invariant holds vacuously. + root: bindings::rb_root::default(), + _p: PhantomData, + } + } + + /// Returns true if this tree is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.root.rb_node.is_null() + } + + /// Returns an iterator over the tree nodes, sorted by key. + pub fn iter(&self) -> Iter<'_, K, V> { + Iter { + _tree: PhantomData, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(&self.root) }, + _phantom: PhantomData, + }, + } + } + + /// Returns a mutable iterator over the tree nodes, sorted by key. + pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { + IterMut { + _tree: PhantomData, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(from_mut(&mut self.root)) }, + _phantom: PhantomData, + }, + } + } + + /// Returns an iterator over the keys of the nodes in the tree, in sorted order. + pub fn keys(&self) -> impl Iterator { + self.iter().map(|(k, _)| k) + } + + /// Returns an iterator over the values of the nodes in the tree, sorted by key. + pub fn values(&self) -> impl Iterator { + self.iter().map(|(_, v)| v) + } + + /// Returns a mutable iterator over the values of the nodes in the tree, sorted by key. + pub fn values_mut(&mut self) -> impl Iterator { + self.iter_mut().map(|(_, v)| v) + } + + /// Returns a cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front_mut(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns an immutable cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back_mut(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } +} + +impl RBTree +where + K: Ord, +{ + /// Tries to insert a new value into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// Returns an error if it cannot allocate memory for the new node. + pub fn try_create_and_insert( + &mut self, + key: K, + value: V, + flags: Flags, + ) -> Result>> { + Ok(self.insert(RBTreeNode::new(key, value, flags)?)) + } + + /// Inserts a new node into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// This function always succeeds. + pub fn insert(&mut self, node: RBTreeNode) -> Option> { + match self.raw_entry(&node.node.key) { + RawEntry::Occupied(entry) => Some(entry.replace(node)), + RawEntry::Vacant(entry) => { + entry.insert(node); + None + } + } + } + + fn raw_entry(&mut self, key: &K) -> RawEntry<'_, K, V> { + let raw_self: *mut RBTree = self; + // The returned `RawEntry` is used to call either `rb_link_node` or `rb_replace_node`. + // The parameters of `bindings::rb_link_node` are as follows: + // - `node`: A pointer to an uninitialized node being inserted. + // - `parent`: A pointer to an existing node in the tree. One of its child pointers must be + // null, and `node` will become a child of `parent` by replacing that child pointer + // with a pointer to `node`. + // - `rb_link`: A pointer to either the left-child or right-child field of `parent`. This + // specifies which child of `parent` should hold `node` after this call. The + // value of `*rb_link` must be null before the call to `rb_link_node`. If the + // red/black tree is empty, then it’s also possible for `parent` to be null. In + // this case, `rb_link` is a pointer to the `root` field of the red/black tree. + // + // We will traverse the tree looking for a node that has a null pointer as its child, + // representing an empty subtree where we can insert our new node. We need to make sure + // that we preserve the ordering of the nodes in the tree. In each iteration of the loop + // we store `parent` and `child_field_of_parent`, and the new `node` will go somewhere + // in the subtree of `parent` that `child_field_of_parent` points at. Once + // we find an empty subtree, we can insert the new node using `rb_link_node`. + let mut parent = core::ptr::null_mut(); + let mut child_field_of_parent: &mut *mut bindings::rb_node = + // SAFETY: `raw_self` is a valid pointer to the `RBTree` (created from `self` above). + unsafe { &mut (*raw_self).root.rb_node }; + while !(*child_field_of_parent).is_null() { + let curr = *child_field_of_parent; + // SAFETY: All links fields we create are in a `Node`. + let node = unsafe { container_of!(curr, Node, links) }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + match key.cmp(unsafe { &(*node).key }) { + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Less => child_field_of_parent = unsafe { &mut (*curr).rb_left }, + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Greater => child_field_of_parent = unsafe { &mut (*curr).rb_right }, + Ordering::Equal => { + return RawEntry::Occupied(OccupiedEntry { + rbtree: self, + node_links: curr, + }) + } + } + parent = curr; + } + + RawEntry::Vacant(RawVacantEntry { + rbtree: raw_self, + parent, + child_field_of_parent, + _phantom: PhantomData, + }) + } + + /// Gets the given key's corresponding entry in the map for in-place manipulation. + pub fn entry(&mut self, key: K) -> Entry<'_, K, V> { + match self.raw_entry(&key) { + RawEntry::Occupied(entry) => Entry::Occupied(entry), + RawEntry::Vacant(entry) => Entry::Vacant(VacantEntry { raw: entry, key }), + } + } + + /// Used for accessing the given node, if it exists. + pub fn find_mut(&mut self, key: &K) -> Option> { + match self.raw_entry(key) { + RawEntry::Occupied(entry) => Some(entry), + RawEntry::Vacant(_entry) => None, + } + } + + /// Returns a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option<&V> { + let mut node = self.root.rb_node; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }; + + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + let this_ref = unsafe { &*this }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let node_ref = unsafe { &*node }; + + node = match key.cmp(&this_ref.key) { + Ordering::Less => node_ref.rb_left, + Ordering::Greater => node_ref.rb_right, + Ordering::Equal => return Some(&this_ref.value), + } + } + None + } + + /// Returns a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + self.find_mut(key).map(|node| node.into_mut()) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the node that was removed if one exists, or [`None`] otherwise. + pub fn remove_node(&mut self, key: &K) -> Option> { + self.find_mut(key).map(OccupiedEntry::remove_node) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the value that was removed if one exists, or [`None`] otherwise. + pub fn remove(&mut self, key: &K) -> Option { + self.find_mut(key).map(OccupiedEntry::remove) + } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound_mut(&mut self, key: &K) -> Option> + where + K: Ord, + { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound(&self, key: &K) -> Option> + where + K: Ord, + { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } + + fn find_best_match(&self, key: &K) -> Option> { + let mut node = self.root.rb_node; + let mut best_key: Option<&K> = None; + let mut best_links: Option> = None; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }; + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + let this_key = unsafe { &(*this).key }; + + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let node_ref = unsafe { &*node }; + + match key.cmp(this_key) { + Ordering::Equal => { + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); + break; + } + Ordering::Greater => { + node = node_ref.rb_right; + } + Ordering::Less => { + let is_better_match = match best_key { + None => true, + Some(best) => best > this_key, + }; + if is_better_match { + best_key = Some(this_key); + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); + } + node = node_ref.rb_left; + } + }; + } + best_links + } +} + +impl Default for RBTree { + fn default() -> Self { + Self::new() + } +} + +impl Drop for RBTree { + fn drop(&mut self) { + // SAFETY: `root` is valid as it's embedded in `self` and we have a valid `self`. + let mut next = unsafe { bindings::rb_first_postorder(&self.root) }; + + // INVARIANT: The loop invariant is that all tree nodes from `next` in postorder are valid. + while !next.is_null() { + // SAFETY: All links fields we create are in a `Node`. + let this = unsafe { container_of!(next, Node, links) }; + + // Find out what the next node is before disposing of the current one. + // SAFETY: `next` and all nodes in postorder are still valid. + next = unsafe { bindings::rb_next_postorder(next) }; + + // INVARIANT: This is the destructor, so we break the type invariant during clean-up, + // but it is not observable. The loop invariant is still maintained. + + // SAFETY: `this` is valid per the loop invariant. + unsafe { drop(KBox::from_raw(this)) }; + } + } +} + +/// A bidirectional mutable cursor over the tree nodes, sorted by key. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Move the cursor, updating it to the 2nd element. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Peek at the next element without impacting the cursor. +/// let next = cursor.peek_next().unwrap(); +/// assert_eq!(next, (&30, &300)); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Moving past the last element causes the cursor to return [`None`]. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// let cursor = cursor.move_next(); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// A cursor can also be obtained at the last element in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// let mut cursor = tree.cursor_back_mut().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Obtaining a cursor returns [`None`] if the tree is empty. +/// +/// ``` +/// use kernel::rbtree::RBTree; +/// +/// let mut tree: RBTree = RBTree::new(); +/// assert!(tree.cursor_front_mut().is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`RBTree::cursor_lower_bound`] can be used to start at an arbitrary node in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert five elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(40, 400, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(50, 500, flags::GFP_KERNEL)?; +/// +/// // If the provided key exists, a cursor to that key is returned. +/// let cursor = tree.cursor_lower_bound(&20).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // If the provided key doesn't exist, a cursor to the first larger element in sort order is returned. +/// let cursor = tree.cursor_lower_bound(&25).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // If there is no larger key, [`None`] is returned. +/// let cursor = tree.cursor_lower_bound(&55); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The cursor allows mutation of values in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Retrieve a cursor. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// +/// // Get a mutable reference to the current value. +/// let (k, v) = cursor.current_mut(); +/// *v = 1000; +/// +/// // The updated value is reflected in the tree. +/// let updated = tree.get(&10).unwrap(); +/// assert_eq!(updated, &1000); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// It also allows node removal. The following examples demonstrate the behavior of removing the current node. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Remove the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// cursor = cursor.remove_current().0.unwrap(); +/// +/// // If a node exists after the current element, it is returned. +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Get a cursor to the last element, and remove it. +/// cursor = tree.cursor_back_mut().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Since there is no next node, the previous node is returned. +/// cursor = cursor.remove_current().0.unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Removing the last element in the tree returns [`None`]. +/// assert!(cursor.remove_current().0.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Nodes adjacent to the current node can also be removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front_mut().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_prev` from the first element returns [`None`]. +/// assert!(cursor.remove_prev().is_none()); +/// +/// // Get a cursor to the last element. +/// cursor = tree.cursor_back_mut().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Calling `remove_prev` removes and returns the middle element. +/// assert_eq!(cursor.remove_prev().unwrap().to_key_value(), (20, 200)); +/// +/// // Calling `remove_next` from the last element returns [`None`]. +/// assert!(cursor.remove_next().is_none()); +/// +/// // Move to the first element +/// cursor = cursor.move_prev().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_next` removes and returns the last element. +/// assert_eq!(cursor.remove_next().unwrap().to_key_value(), (30, 300)); +/// +/// # Ok::<(), Error>(()) +/// +/// ``` +/// +/// # Invariants +/// - `current` points to a node that is in the same [`RBTree`] as `tree`. +pub struct CursorMut<'a, K, V> { + tree: &'a mut RBTree, + current: NonNull, +} + +/// A bidirectional immutable cursor over the tree nodes, sorted by key. This is a simpler +/// variant of [`CursorMut`] that is basically providing read only access. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let cursor = tree.cursor_front().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct Cursor<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + current: NonNull, +} + +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. +unsafe impl<'a, K: Sync, V: Sync> Send for Cursor<'a, K, V> {} + +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. +unsafe impl<'a, K: Sync, V: Sync> Sync for Cursor<'a, K, V> {} + +impl<'a, K, V> Cursor<'a, K, V> { + /// The current node + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of `'b`. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { &(*this).value }; + (k, v) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the next node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } +} + +// SAFETY: The [`CursorMut`] has exclusive access to both `K` and `V`, so it is sufficient to +// require them to be `Send`. +// The cursor only gives out immutable references to the keys, but since it has exclusive access to +// those same keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the +// user. +unsafe impl<'a, K: Send, V: Send> Send for CursorMut<'a, K, V> {} + +// SAFETY: The [`CursorMut`] gives out immutable references to `K` and mutable references to `V`, +// so it has the same thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for CursorMut<'a, K, V> {} + +impl<'a, K, V> CursorMut<'a, K, V> { + /// The current node. + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + + /// The current node, with a mutable value + pub fn current_mut(&mut self) -> (&K, &mut V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an mutable reference by the function signature. + unsafe { Self::to_key_value_mut(self.current) } + } + + /// Remove the current node from the tree. + /// + /// Returns a tuple where the first element is a cursor to the next node, if it exists, + /// else the previous node, else [`None`] (if the tree becomes empty). The second element + /// is the removed node. + pub fn remove_current(self) -> (Option, RBTreeNode) { + let prev = self.get_neighbor_raw(Direction::Prev); + let next = self.get_neighbor_raw(Direction::Next); + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(self.current.as_ptr(), Node, links) }; + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { KBox::from_raw(this) }; + let node = RBTreeNode { node }; + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(&mut (*this).links, addr_of_mut!(self.tree.root)) }; + + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self.tree`. + let cursor = next.or(prev).map(|current| Self { + current, + tree: self.tree, + }); + + (cursor, node) + } + + /// Remove the previous node, returning it if it exists. + pub fn remove_prev(&mut self) -> Option> { + self.remove_neighbor(Direction::Prev) + } + + /// Remove the next node, returning it if it exists. + pub fn remove_next(&mut self) -> Option> { + self.remove_neighbor(Direction::Next) + } + + fn remove_neighbor(&mut self, direction: Direction) -> Option> { + if let Some(neighbor) = self.get_neighbor_raw(direction) { + let neighbor = neighbor.as_ptr(); + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(neighbor, addr_of_mut!(self.tree.root)) }; + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(neighbor, Node, links) }; + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { KBox::from_raw(this) }; + return Some(RBTreeNode { node }); + } + None + } + + /// Move the cursor to the previous node, returning [`None`] if it doesn't exist. + pub fn move_prev(self) -> Option { + self.mv(Direction::Prev) + } + + /// Move the cursor to the next node, returning [`None`] if it doesn't exist. + pub fn move_next(self) -> Option { + self.mv(Direction::Next) + } + + fn mv(self, direction: Direction) -> Option { + // INVARIANT: + // - `neighbor` is a valid node in the [`RBTree`] pointed to by `self.tree`. + self.get_neighbor_raw(direction).map(|neighbor| Self { + tree: self.tree, + current: neighbor, + }) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the next node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + /// Access the previous node mutably without moving the cursor. + pub fn peek_prev_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Prev) + } + + /// Access the next node mutably without moving the cursor. + pub fn peek_next_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Next) + } + + fn peek_mut(&mut self, direction: Direction) -> Option<(&K, &mut V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have a mutable reference to `self`. + unsafe { Self::to_key_value_mut(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of `'b`. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees immutable access to `node`. + (k, unsafe { &*v }) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has mutable access to `node` for the duration of `'b`. + unsafe fn to_key_value_mut<'b>(node: NonNull) -> (&'b K, &'b mut V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees mutable access to `node`. + (k, unsafe { &mut *v }) + } + + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to the key for the duration of `'b`. + unsafe fn to_key_value_raw<'b>(node: NonNull) -> (&'b K, *mut V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { addr_of_mut!((*this).value) }; + (k, v) + } +} + +/// Direction for [`Cursor`] and [`CursorMut`] operations. +enum Direction { + /// the node immediately before, in sort order + Prev, + /// the node immediately after, in sort order + Next, +} + +impl<'a, K, V> IntoIterator for &'a RBTree { + type Item = (&'a K, &'a V); + type IntoIter = Iter<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter`]. +pub struct Iter<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + iter_raw: IterRaw, +} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Send for Iter<'a, K, V> {} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for Iter<'a, K, V> {} + +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + // SAFETY: Due to `self._tree`, `k` and `v` are valid for the lifetime of `'a`. + self.iter_raw.next().map(|(k, v)| unsafe { (&*k, &*v) }) + } +} + +impl<'a, K, V> IntoIterator for &'a mut RBTree { + type Item = (&'a K, &'a mut V); + type IntoIter = IterMut<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// A mutable iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter_mut`]. +pub struct IterMut<'a, K, V> { + _tree: PhantomData<&'a mut RBTree>, + iter_raw: IterRaw, +} + +// SAFETY: The [`IterMut`] has exclusive access to both `K` and `V`, so it is sufficient to require them to be `Send`. +// The iterator only gives out immutable references to the keys, but since the iterator has exclusive access to those same +// keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the user. +unsafe impl<'a, K: Send, V: Send> Send for IterMut<'a, K, V> {} + +// SAFETY: The [`IterMut`] gives out immutable references to K and mutable references to V, so it has the same +// thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for IterMut<'a, K, V> {} + +impl<'a, K, V> Iterator for IterMut<'a, K, V> { + type Item = (&'a K, &'a mut V); + + fn next(&mut self) -> Option { + self.iter_raw.next().map(|(k, v)| + // SAFETY: Due to `&mut self`, we have exclusive access to `k` and `v`, for the lifetime of `'a`. + unsafe { (&*k, &mut *v) }) + } +} + +/// A raw iterator over the nodes of a [`RBTree`]. +/// +/// # Invariants +/// - `self.next` is a valid pointer. +/// - `self.next` points to a node stored inside of a valid `RBTree`. +struct IterRaw { + next: *mut bindings::rb_node, + _phantom: PhantomData (K, V)>, +} + +impl Iterator for IterRaw { + type Item = (*mut K, *mut V); + + fn next(&mut self) -> Option { + if self.next.is_null() { + return None; + } + + // SAFETY: By the type invariant of `IterRaw`, `self.next` is a valid node in an `RBTree`, + // and by the type invariant of `RBTree`, all nodes point to the links field of `Node` objects. + let cur = unsafe { container_of!(self.next, Node, links) }; + + // SAFETY: `self.next` is a valid tree node by the type invariants. + self.next = unsafe { bindings::rb_next(self.next) }; + + // SAFETY: By the same reasoning above, it is safe to dereference the node. + Some(unsafe { (addr_of_mut!((*cur).key), addr_of_mut!((*cur).value)) }) + } +} + +/// A memory reservation for a red-black tree node. +/// +/// +/// It contains the memory needed to hold a node that can be inserted into a red-black tree. One +/// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). +pub struct RBTreeNodeReservation { + node: KBox>>, +} + +impl RBTreeNodeReservation { + /// Allocates memory for a node to be eventually initialised and inserted into the tree via a + /// call to [`RBTree::insert`]. + pub fn new(flags: Flags) -> Result> { + Ok(RBTreeNodeReservation { + node: KBox::new_uninit(flags)?, + }) + } +} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. Those can always +// be moved across threads. +unsafe impl Send for RBTreeNodeReservation {} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. +unsafe impl Sync for RBTreeNodeReservation {} + +impl RBTreeNodeReservation { + /// Initialises a node reservation. + /// + /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. + pub fn into_node(self, key: K, value: V) -> RBTreeNode { + let node = KBox::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); + RBTreeNode { node } + } +} + +/// A red-black tree node. +/// +/// The node is fully initialised (with key and value) and can be inserted into a tree without any +/// extra allocations or failure paths. +pub struct RBTreeNode { + node: KBox>, +} + +impl RBTreeNode { + /// Allocates and initialises a node that can be inserted into the tree via + /// [`RBTree::insert`]. + pub fn new(key: K, value: V, flags: Flags) -> Result> { + Ok(RBTreeNodeReservation::new(flags)?.into_node(key, value)) + } + + /// Get the key and value from inside the node. + pub fn to_key_value(self) -> (K, V) { + let node = KBox::into_inner(self.node); + + (node.key, node.value) + } +} + +// SAFETY: If K and V can be sent across threads, then it's also okay to send [`RBTreeNode`] across +// threads. +unsafe impl Send for RBTreeNode {} + +// SAFETY: If K and V can be accessed without synchronization, then it's also okay to access +// [`RBTreeNode`] without synchronization. +unsafe impl Sync for RBTreeNode {} + +impl RBTreeNode { + /// Drop the key and value, but keep the allocation. + /// + /// It then becomes a reservation that can be re-initialised into a different node (i.e., with + /// a different key and/or value). + /// + /// The existing key and value are dropped in-place as part of this operation, that is, memory + /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). + pub fn into_reservation(self) -> RBTreeNodeReservation { + RBTreeNodeReservation { + node: KBox::drop_contents(self.node), + } + } +} + +/// A view into a single entry in a map, which may either be vacant or occupied. +/// +/// This enum is constructed from the [`RBTree::entry`]. +/// +/// [`entry`]: fn@RBTree::entry +pub enum Entry<'a, K, V> { + /// This [`RBTree`] does not have a node with this key. + Vacant(VacantEntry<'a, K, V>), + /// This [`RBTree`] already has a node with this key. + Occupied(OccupiedEntry<'a, K, V>), +} + +/// Like [`Entry`], except that it doesn't have ownership of the key. +enum RawEntry<'a, K, V> { + Vacant(RawVacantEntry<'a, K, V>), + Occupied(OccupiedEntry<'a, K, V>), +} + +/// A view into a vacant entry in a [`RBTree`]. It is part of the [`Entry`] enum. +pub struct VacantEntry<'a, K, V> { + key: K, + raw: RawVacantEntry<'a, K, V>, +} + +/// Like [`VacantEntry`], but doesn't hold on to the key. +/// +/// # Invariants +/// - `parent` may be null if the new node becomes the root. +/// - `child_field_of_parent` is a valid pointer to the left-child or right-child of `parent`. If `parent` is +/// null, it is a pointer to the root of the [`RBTree`]. +struct RawVacantEntry<'a, K, V> { + rbtree: *mut RBTree, + /// The node that will become the parent of the new node if we insert one. + parent: *mut bindings::rb_node, + /// This points to the left-child or right-child field of `parent`, or `root` if `parent` is + /// null. + child_field_of_parent: *mut *mut bindings::rb_node, + _phantom: PhantomData<&'a mut RBTree>, +} + +impl<'a, K, V> RawVacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + /// + /// The `node` must have a key such that inserting it here does not break the ordering of this + /// [`RBTree`]. + fn insert(self, node: RBTreeNode) -> &'a mut V { + let node = KBox::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when + // the node is removed or replaced. + let node_links = unsafe { addr_of_mut!((*node).links) }; + + // INVARIANT: We are linking in a new node, which is valid. It remains valid because we + // "forgot" it with `KBox::into_raw`. + // SAFETY: The type invariants of `RawVacantEntry` are exactly the safety requirements of `rb_link_node`. + unsafe { bindings::rb_link_node(node_links, self.parent, self.child_field_of_parent) }; + + // SAFETY: All pointers are valid. `node` has just been inserted into the tree. + unsafe { bindings::rb_insert_color(node_links, addr_of_mut!((*self.rbtree).root)) }; + + // SAFETY: The node is valid until we remove it from the tree. + unsafe { &mut (*node).value } + } +} + +impl<'a, K, V> VacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + pub fn insert(self, value: V, reservation: RBTreeNodeReservation) -> &'a mut V { + self.raw.insert(reservation.into_node(self.key, value)) + } +} + +/// A view into an occupied entry in a [`RBTree`]. It is part of the [`Entry`] enum. +/// +/// # Invariants +/// - `node_links` is a valid, non-null pointer to a tree node in `self.rbtree` +pub struct OccupiedEntry<'a, K, V> { + rbtree: &'a mut RBTree, + /// The node that this entry corresponds to. + node_links: *mut bindings::rb_node, +} + +impl<'a, K, V> OccupiedEntry<'a, K, V> { + /// Gets a reference to the value in the entry. + pub fn get(&self) -> &V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have shared access to the underlying tree, and can thus give out a shared reference. + unsafe { &(*container_of!(self.node_links, Node, links)).value } + } + + /// Gets a mutable reference to the value in the entry. + pub fn get_mut(&mut self) -> &mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have exclusive access to the underlying tree, and can thus give out a mutable reference. + unsafe { &mut (*(container_of!(self.node_links, Node, links))).value } + } + + /// Converts the entry into a mutable reference to its value. + /// + /// If you need multiple references to the `OccupiedEntry`, see [`self#get_mut`]. + pub fn into_mut(self) -> &'a mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - This consumes the `&'a mut RBTree`, therefore it can give out a mutable reference that lives for `'a`. + unsafe { &mut (*(container_of!(self.node_links, Node, links))).value } + } + + /// Remove this entry from the [`RBTree`]. + pub fn remove_node(self) -> RBTreeNode { + // SAFETY: The node is a node in the tree, so it is valid. + unsafe { bindings::rb_erase(self.node_links, &mut self.rbtree.root) }; + + // INVARIANT: The node is being returned and the caller may free it, however, it was + // removed from the tree. So the invariants still hold. + RBTreeNode { + // SAFETY: The node was a node in the tree, but we removed it, so we can convert it + // back into a box. + node: unsafe { KBox::from_raw(container_of!(self.node_links, Node, links)) }, + } + } + + /// Takes the value of the entry out of the map, and returns it. + pub fn remove(self) -> V { + let rb_node = self.remove_node(); + let node = KBox::into_inner(rb_node.node); + + node.value + } + + /// Swap the current node for the provided node. + /// + /// The key of both nodes must be equal. + fn replace(self, node: RBTreeNode) -> RBTreeNode { + let node = KBox::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when + // the node is removed or replaced. + let new_node_links = unsafe { addr_of_mut!((*node).links) }; + + // SAFETY: This updates the pointers so that `new_node_links` is in the tree where + // `self.node_links` used to be. + unsafe { + bindings::rb_replace_node(self.node_links, new_node_links, &mut self.rbtree.root) + }; + + // SAFETY: + // - `self.node_ptr` produces a valid pointer to a node in the tree. + // - Now that we removed this entry from the tree, we can convert the node to a box. + let old_node = unsafe { KBox::from_raw(container_of!(self.node_links, Node, links)) }; + + RBTreeNode { node: old_node } + } +} + +struct Node { + links: bindings::rb_node, + key: K, + value: V, +} diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 000000000..41e730ced --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Regulator abstractions, providing a standard kernel interface to control +//! voltage and current regulators. +//! +//! The intention is to allow systems to dynamically control regulator power +//! output in order to save power and prolong battery life. This applies to both +//! voltage regulators (where voltage output is controllable) and current sinks +//! (where current limit is controllable). +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Regulators are modeled in Rust with a collection of states. Each state may +//! enforce a given invariant, and they may convert between each other where applicable. +//! +//! See [Voltage and current regulator API](https://docs.kernel.org/driver-api/regulator.html) +//! for more information. + +use crate::{ + bindings, + device::{Bound, Device}, + error::{from_err_ptr, to_result, Result}, + prelude::*, +}; + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, // +}; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Enabled {} + impl Sealed for super::Disabled {} +} + +/// A trait representing the different states a [`Regulator`] can be in. +pub trait RegulatorState: private::Sealed + 'static { + /// Whether the regulator should be disabled when dropped. + const DISABLE_ON_DROP: bool; +} + +/// A state where the [`Regulator`] is known to be enabled. +/// +/// The `enable` reference count held by this state is decremented when it is +/// dropped. +pub struct Enabled; + +/// A state where this [`Regulator`] handle has not specifically asked for the +/// underlying regulator to be enabled. This means that this reference does not +/// own an `enable` reference count, but the regulator may still be on. +pub struct Disabled; + +impl RegulatorState for Enabled { + const DISABLE_ON_DROP: bool = true; +} + +impl RegulatorState for Disabled { + const DISABLE_ON_DROP: bool = false; +} + +/// A trait that abstracts the ability to check if a [`Regulator`] is enabled. +pub trait IsEnabled: RegulatorState {} +impl IsEnabled for Disabled {} + +/// An error that can occur when trying to convert a [`Regulator`] between states. +pub struct Error { + /// The error that occurred. + pub error: kernel::error::Error, + + /// The regulator that caused the error, so that the operation may be retried. + pub regulator: Regulator, +} +/// Obtains and enables a [`devres`]-managed regulator for a device. +/// +/// This calls [`regulator_disable()`] and [`regulator_put()`] automatically on +/// driver detach. +/// +/// This API is identical to `devm_regulator_get_enable()`, and should be +/// preferred over the [`Regulator`] API if the caller only +/// cares about the regulator being enabled. +/// +/// [`devres`]: https://docs.kernel.org/driver-api/driver-model/devres.html +/// [`regulator_disable()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_disable +/// [`regulator_put()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_put +pub fn devm_enable(dev: &Device, name: &CStr) -> Result { + // SAFETY: `dev` is a valid and bound device, while `name` is a valid C + // string. + to_result(unsafe { bindings::devm_regulator_get_enable(dev.as_raw(), name.as_char_ptr()) }) +} + +/// Same as [`devm_enable`], but calls `devm_regulator_get_enable_optional` +/// instead. +/// +/// This obtains and enables a [`devres`]-managed regulator for a device, but +/// does not print a message nor provides a dummy if the regulator is not found. +/// +/// This calls [`regulator_disable()`] and [`regulator_put()`] automatically on +/// driver detach. +/// +/// [`devres`]: https://docs.kernel.org/driver-api/driver-model/devres.html +/// [`regulator_disable()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_disable +/// [`regulator_put()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_put +pub fn devm_enable_optional(dev: &Device, name: &CStr) -> Result { + // SAFETY: `dev` is a valid and bound device, while `name` is a valid C + // string. + to_result(unsafe { + bindings::devm_regulator_get_enable_optional(dev.as_raw(), name.as_char_ptr()) + }) +} + +/// A `struct regulator` abstraction. +/// +/// # Examples +/// +/// ## Enabling a regulator +/// +/// This example uses [`Regulator`], which is suitable for drivers that +/// enable a regulator at probe time and leave them on until the device is +/// removed or otherwise shutdown. +/// +/// These users can store [`Regulator`] directly in their driver's +/// private data struct. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Disabled, Enabled}; +/// fn enable(dev: &Device, min_voltage: Voltage, max_voltage: Voltage) -> Result { +/// // Obtain a reference to a (fictitious) regulator. +/// let regulator: Regulator = Regulator::::get(dev, c"vcc")?; +/// +/// // The voltage can be set before enabling the regulator if needed, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Enables the regulator, consuming the previous value. +/// // +/// // From now on, the regulator is known to be enabled because of the type +/// // `Enabled`. +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator = +/// regulator.try_into_enabled().map_err(|error| error.error)?; +/// +/// // The voltage can also be set after enabling the regulator, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// A more concise shortcut is available for enabling a regulator. This is +/// equivalent to `regulator_get_enable()`: +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Enabled}; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. +/// let regulator: Regulator = Regulator::::get(dev, c"vcc")?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// If a driver only cares about the regulator being on for as long it is bound +/// to a device, then it should use [`devm_enable`] or [`devm_enable_optional`]. +/// This should be the default use-case unless more fine-grained control over +/// the regulator's state is required. +/// +/// [`devm_enable`]: crate::regulator::devm_enable +/// [`devm_optional`]: crate::regulator::devm_enable_optional +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::{Bound, Device}; +/// # use kernel::regulator; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. This +/// // call only returns whether the operation succeeded. +/// regulator::devm_enable(dev, c"vcc")?; +/// +/// // The regulator will be disabled and put when `dev` is unbound. +/// Ok(()) +/// } +/// ``` +/// +/// ## Disabling a regulator +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Enabled, Disabled}; +/// fn disable(dev: &Device, regulator: Regulator) -> Result { +/// // We can also disable an enabled regulator without reliquinshing our +/// // refcount: +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator = +/// regulator.try_into_disabled().map_err(|error| error.error)?; +/// +/// // The refcount will be decremented when `regulator` is dropped. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// # Invariants +/// +/// - `inner` is a pointer obtained from a successful call to +/// [`regulator_get()`]. It is treated as an opaque token that may only be +/// accessed using C API methods (e.g., it may be `NULL` if the C API returns +/// `NULL`). +/// +/// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get +pub struct Regulator +where + State: RegulatorState, +{ + inner: *mut bindings::regulator, + _phantom: PhantomData, +} + +impl Regulator { + /// Sets the voltage for the regulator. + /// + /// This can be used to ensure that the device powers up cleanly. + pub fn set_voltage(&self, min_voltage: Voltage, max_voltage: Voltage) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { + bindings::regulator_set_voltage( + self.inner, + min_voltage.as_microvolts(), + max_voltage.as_microvolts(), + ) + }) + } + + /// Gets the current voltage of the regulator. + pub fn get_voltage(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + let voltage = unsafe { bindings::regulator_get_voltage(self.inner) }; + + to_result(voltage).map(|()| Voltage::from_microvolts(voltage)) + } + + fn get_internal(dev: &Device, name: &CStr) -> Result> { + let inner = + // SAFETY: It is safe to call `regulator_get()`, on a device pointer + // received from the C code. + from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_char_ptr()) })?; + + // INVARIANT: `inner` is a pointer obtained from `regulator_get()`, and + // the call was successful. + Ok(Self { + inner, + _phantom: PhantomData, + }) + } + + fn enable_internal(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_enable(self.inner) }) + } + + fn disable_internal(&self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_disable(self.inner) }) + } +} + +impl Regulator { + /// Obtains a [`Regulator`] instance from the system. + pub fn get(dev: &Device, name: &CStr) -> Result { + Regulator::get_internal(dev, name) + } + + /// Attempts to convert the regulator to an enabled state. + pub fn try_into_enabled(self) -> Result, Error> { + // We will be transferring the ownership of our `regulator_get()` count to + // `Regulator`. + let regulator = ManuallyDrop::new(self); + + regulator + .enable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator { + /// Obtains a [`Regulator`] instance from the system and enables it. + /// + /// This is equivalent to calling `regulator_get_enable()` in the C API. + pub fn get(dev: &Device, name: &CStr) -> Result { + Regulator::::get_internal(dev, name)? + .try_into_enabled() + .map_err(|error| error.error) + } + + /// Attempts to convert the regulator to a disabled state. + pub fn try_into_disabled(self) -> Result, Error> { + // We will be transferring the ownership of our `regulator_get()` count + // to `Regulator`. + let regulator = ManuallyDrop::new(self); + + regulator + .disable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator { + /// Checks if the regulator is enabled. + pub fn is_enabled(&self) -> bool { + // SAFETY: Safe as per the type invariants of `Regulator`. + unsafe { bindings::regulator_is_enabled(self.inner) != 0 } + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + if T::DISABLE_ON_DROP { + // SAFETY: By the type invariants, we know that `self` owns a + // reference on the enabled refcount, so it is safe to relinquish it + // now. + unsafe { bindings::regulator_disable(self.inner) }; + } + // SAFETY: By the type invariants, we know that `self` owns a reference, + // so it is safe to relinquish it now. + unsafe { bindings::regulator_put(self.inner) }; + } +} + +// SAFETY: It is safe to send a `Regulator` across threads. In particular, a +// Regulator can be dropped from any thread. +unsafe impl Send for Regulator {} + +// SAFETY: It is safe to send a &Regulator across threads because the C side +// handles its own locking. +unsafe impl Sync for Regulator {} + +/// A voltage. +/// +/// This type represents a voltage value in microvolts. +#[repr(transparent)] +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct Voltage(i32); + +impl Voltage { + /// Creates a new `Voltage` from a value in microvolts. + pub fn from_microvolts(uv: i32) -> Self { + Self(uv) + } + + /// Returns the value of the voltage in microvolts as an [`i32`]. + pub fn as_microvolts(self) -> i32 { + self.0 + } +} diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs new file mode 100644 index 000000000..0e55e2a0f --- /dev/null +++ b/rust/kernel/revocable.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Revocable objects. +//! +//! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence +//! of a [`RevocableGuard`] ensures that objects remain valid. + +use pin_init::Wrapper; + +use crate::{ + prelude::*, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::drop_in_place, // +}; + +/// An object that can become inaccessible at runtime. +/// +/// Once access is revoked and all concurrent users complete (i.e., all existing instances of +/// [`RevocableGuard`] are dropped), the wrapped object is also dropped. +/// +/// # Examples +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = v.try_access()?; +/// Some(guard.a + guard.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +/// +/// Sample example as above, but explicitly using the rcu read side lock. +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// use kernel::sync::rcu; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = rcu::read_lock(); +/// let e = v.try_access_with_guard(&guard)?; +/// Some(e.a + e.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +#[pin_data(PinnedDrop)] +pub struct Revocable { + is_available: AtomicFlag, + #[pin] + data: Opaque, +} + +// SAFETY: `Revocable` is `Send` if the wrapped object is also `Send`. This is because while the +// functionality exposed by `Revocable` can be accessed from any thread/CPU, it is possible that +// this isn't supported by the wrapped object. +unsafe impl Send for Revocable {} + +// SAFETY: `Revocable` is `Sync` if the wrapped object is both `Send` and `Sync`. We require `Send` +// from the wrapped object as well because of `Revocable::revoke`, which can trigger the `Drop` +// implementation of the wrapped object from an arbitrary thread. +unsafe impl Sync for Revocable {} + +impl Revocable { + /// Creates a new revocable instance of the given data. + pub fn new(data: impl PinInit) -> impl PinInit { + try_pin_init!(Self { + is_available: AtomicFlag::new(true), + data <- Opaque::pin_init(data), + }? E) + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a guard that gives access to the object otherwise; the object is guaranteed to + /// remain accessible while the guard is alive. In such cases, callers are not allowed to sleep + /// because another CPU may be waiting to complete the revocation of this object. + pub fn try_access(&self) -> Option> { + let guard = rcu::read_lock(); + if self.is_available.load(Relaxed) { + // Since `self.is_available` is true, data is initialised and has to remain valid + // because the RCU read side lock prevents it from being dropped. + Some(RevocableGuard::new(self.data.get(), guard)) + } else { + None + } + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a shared reference to the object otherwise; the object is guaranteed to + /// remain accessible while the rcu read side guard is alive. In such cases, callers are not + /// allowed to sleep because another CPU may be waiting to complete the revocation of this + /// object. + pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { + if self.is_available.load(Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { &*self.data.get() }) + } else { + None + } + } + + /// Tries to access the wrapped object and run a closure on it while the guard is held. + /// + /// This is a convenience method to run short non-sleepable code blocks while ensuring the + /// guard is dropped afterwards. [`Self::try_access`] carries the risk that the caller will + /// forget to explicitly drop that returned guard before calling sleepable code; this method + /// adds an extra safety to make sure it doesn't happen. + /// + /// Returns [`None`] if the object has been revoked and is therefore no longer accessible, or + /// the result of the closure wrapped in [`Some`]. If the closure returns a [`Result`] then the + /// return type becomes `Option>`, which can be inconvenient. Users are encouraged to + /// define their own macro that turns the [`Option`] into a proper error code and flattens the + /// inner result into it if it makes sense within their subsystem. + pub fn try_access_with R>(&self, f: F) -> Option { + self.try_access().map(|t| f(&*t)) + } + + /// Directly access the revocable wrapped object. + /// + /// # Safety + /// + /// The caller must ensure this [`Revocable`] instance hasn't been revoked and won't be revoked + /// as long as the returned `&T` lives. + pub unsafe fn access(&self) -> &T { + // SAFETY: By the safety requirement of this function it is guaranteed that + // `self.data.get()` is a valid pointer to an instance of `T`. + unsafe { &*self.data.get() } + } + + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + unsafe fn revoke_internal(&self) -> bool { + let revoke = self.is_available.xchg(false, Relaxed); + + if revoke { + if SYNC { + rcu::synchronize_rcu(); + } + + // SAFETY: We know `self.data` is valid because only one CPU can succeed the + // `compare_exchange` above that takes `is_available` from `true` to `false`. + unsafe { drop_in_place(self.data.get()) }; + } + + revoke + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`], + /// expecting that there are no concurrent users of the object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + /// + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + pub unsafe fn revoke_nosync(&self) -> bool { + // SAFETY: By the safety requirement of this function, the caller ensures that nobody is + // accessing the data anymore and hence we don't have to wait for the grace period to + // finish. + unsafe { self.revoke_internal::() } + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`]. + /// + /// If there are concurrent users of the object (i.e., ones that called + /// [`Revocable::try_access`] beforehand and still haven't dropped the returned guard), this + /// function waits for the concurrent access to complete before dropping the wrapped object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + pub fn revoke(&self) -> bool { + // SAFETY: By passing `true` we ask `revoke_internal` to wait for the grace period to + // finish. + unsafe { self.revoke_internal::() } + } +} + +#[pinned_drop] +impl PinnedDrop for Revocable { + fn drop(self: Pin<&mut Self>) { + // Drop only if the data hasn't been revoked yet (in which case it has already been + // dropped). + // SAFETY: We are not moving out of `p`, only dropping in place + let p = unsafe { self.get_unchecked_mut() }; + if *p.is_available.get_mut() { + // SAFETY: We know `self.data` is valid because no other CPU has changed + // `is_available` to `false` yet, and no other CPU can do it anymore because this CPU + // holds the only reference (mutable) to `self` now. + unsafe { drop_in_place(p.data.get()) }; + } + } +} + +/// A guard that allows access to a revocable object and keeps it alive. +/// +/// CPUs may not sleep while holding on to [`RevocableGuard`] because it's in atomic context +/// holding the RCU read-side lock. +/// +/// # Invariants +/// +/// The RCU read-side lock is held while the guard is alive. +pub struct RevocableGuard<'a, T> { + // This can't use the `&'a T` type because references that appear in function arguments must + // not become dangling during the execution of the function, which can happen if the + // `RevocableGuard` is passed as a function argument and then dropped during execution of the + // function. + data_ref: *const T, + _rcu_guard: rcu::Guard, + _p: PhantomData<&'a ()>, +} + +impl RevocableGuard<'_, T> { + fn new(data_ref: *const T, rcu_guard: rcu::Guard) -> Self { + Self { + data_ref, + _rcu_guard: rcu_guard, + _p: PhantomData, + } + } +} + +impl Deref for RevocableGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariants, we hold the rcu read-side lock, so the object is + // guaranteed to remain valid. + unsafe { &*self.data_ref } + } +} diff --git a/rust/kernel/safety.rs b/rust/kernel/safety.rs new file mode 100644 index 000000000..c1c6bd0fa --- /dev/null +++ b/rust/kernel/safety.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Safety related APIs. + +/// Checks that a precondition of an unsafe function is followed. +/// +/// The check is enabled at runtime if debug assertions (`CONFIG_RUST_DEBUG_ASSERTIONS`) +/// are enabled. Otherwise, this macro is a no-op. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::unsafe_precondition_assert; +/// +/// struct RawBuffer { +/// data: [T; N], +/// } +/// +/// impl RawBuffer { +/// /// # Safety +/// /// +/// /// The caller must ensure that `index` is less than `N`. +/// unsafe fn set_unchecked(&mut self, index: usize, value: T) { +/// unsafe_precondition_assert!( +/// index < N, +/// "RawBuffer::set_unchecked() requires index ({index}) < N ({N})" +/// ); +/// +/// // SAFETY: By the safety requirements of this function, `index` is valid. +/// unsafe { +/// *self.data.get_unchecked_mut(index) = value; +/// } +/// } +/// } +/// ``` +/// +/// # Panics +/// +/// Panics if the expression is evaluated to [`false`] at runtime. +#[macro_export] +macro_rules! unsafe_precondition_assert { + ($cond:expr $(,)?) => { + $crate::unsafe_precondition_assert!(@inner $cond, ::core::stringify!($cond)) + }; + + ($cond:expr, $($arg:tt)+) => { + $crate::unsafe_precondition_assert!(@inner $cond, $crate::prelude::fmt!($($arg)+)) + }; + + (@inner $cond:expr, $msg:expr) => { + ::core::debug_assert!($cond, "unsafe precondition violated: {}", $msg) + }; +} diff --git a/rust/kernel/scatterlist.rs b/rust/kernel/scatterlist.rs new file mode 100644 index 000000000..b83c468b5 --- /dev/null +++ b/rust/kernel/scatterlist.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for scatter-gather lists. +//! +//! C header: [`include/linux/scatterlist.h`](srctree/include/linux/scatterlist.h) +//! +//! Scatter-gather (SG) I/O is a memory access technique that allows devices to perform DMA +//! operations on data buffers that are not physically contiguous in memory. It works by creating a +//! "scatter-gather list", an array where each entry specifies the address and length of a +//! physically contiguous memory segment. +//! +//! The device's DMA controller can then read this list and process the segments sequentially as +//! part of one logical I/O request. This avoids the need for a single, large, physically contiguous +//! memory buffer, which can be difficult or impossible to allocate. +//! +//! This module provides safe Rust abstractions over the kernel's `struct scatterlist` and +//! `struct sg_table` types. +//! +//! The main entry point is the [`SGTable`] type, which represents a complete scatter-gather table. +//! It can be either: +//! +//! - An owned table ([`SGTable>`]), created from a Rust memory buffer (e.g., [`VVec`]). +//! This type manages the allocation of the `struct sg_table`, the DMA mapping of the buffer, and +//! the automatic cleanup of all resources. +//! - A borrowed reference (&[`SGTable`]), which provides safe, read-only access to a table that was +//! allocated by other (e.g., C) code. +//! +//! Individual entries in the table are represented by [`SGEntry`], which can be accessed by +//! iterating over an [`SGTable`]. + +use crate::{ + alloc, + alloc::allocator::VmallocPageIter, + bindings, + device::{Bound, Device}, + devres::Devres, + dma, error, + io::ResourceSize, + page, + prelude::*, + sync::aref::ARef, + types::Opaque, +}; +use core::{ops::Deref, ptr::NonNull}; + +/// A single entry in a scatter-gather list. +/// +/// An `SGEntry` represents a single, physically contiguous segment of memory that has been mapped +/// for DMA. +/// +/// Instances of this struct are obtained by iterating over an [`SGTable`]. Drivers do not create +/// or own [`SGEntry`] objects directly. +#[repr(transparent)] +pub struct SGEntry(Opaque); + +// SAFETY: `SGEntry` can be sent to any task. +unsafe impl Send for SGEntry {} + +// SAFETY: `SGEntry` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for SGEntry {} + +impl SGEntry { + /// Convert a raw `struct scatterlist *` to a `&'a SGEntry`. + /// + /// # Safety + /// + /// Callers must ensure that the `struct scatterlist` pointed to by `ptr` is valid for the + /// lifetime `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::scatterlist) -> &'a Self { + // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer + // to a `struct scatterlist` for the duration of `'a`. + unsafe { &*ptr.cast() } + } + + /// Obtain the raw `struct scatterlist *`. + #[inline] + fn as_raw(&self) -> *mut bindings::scatterlist { + self.0.get() + } + + /// Returns the DMA address of this SG entry. + /// + /// This is the address that the device should use to access the memory segment. + #[inline] + pub fn dma_address(&self) -> dma::DmaAddress { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. + unsafe { bindings::sg_dma_address(self.as_raw()) } + } + + /// Returns the length of this SG entry in bytes. + #[inline] + pub fn dma_len(&self) -> ResourceSize { + #[allow(clippy::useless_conversion)] + // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. + unsafe { bindings::sg_dma_len(self.as_raw()) }.into() + } +} + +/// The borrowed generic type of an [`SGTable`], representing a borrowed or externally managed +/// table. +#[repr(transparent)] +pub struct Borrowed(Opaque); + +// SAFETY: `Borrowed` can be sent to any task. +unsafe impl Send for Borrowed {} + +// SAFETY: `Borrowed` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for Borrowed {} + +/// A scatter-gather table. +/// +/// This struct is a wrapper around the kernel's `struct sg_table`. It manages a list of DMA-mapped +/// memory segments that can be passed to a device for I/O operations. +/// +/// The generic parameter `T` is used as a generic type to distinguish between owned and borrowed +/// tables. +/// +/// - [`SGTable`]: An owned table created and managed entirely by Rust code. It handles +/// allocation, DMA mapping, and cleanup of all associated resources. See [`SGTable::new`]. +/// - [`SGTable`} (or simply [`SGTable`]): Represents a table whose lifetime is managed +/// externally. It can be used safely via a borrowed reference `&'a SGTable`, where `'a` is the +/// external lifetime. +/// +/// All [`SGTable`] variants can be iterated over the individual [`SGEntry`]s. +#[repr(transparent)] +#[pin_data] +pub struct SGTable { + #[pin] + inner: T, +} + +impl SGTable { + /// Creates a borrowed `&'a SGTable` from a raw `struct sg_table` pointer. + /// + /// This allows safe access to an `sg_table` that is managed elsewhere (for example, in C code). + /// + /// # Safety + /// + /// Callers must ensure that: + /// + /// - the `struct sg_table` pointed to by `ptr` is valid for the entire lifetime of `'a`, + /// - the data behind `ptr` is not modified concurrently for the duration of `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::sg_table) -> &'a Self { + // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer + // to a `struct sg_table` for the duration of `'a`. + unsafe { &*ptr.cast() } + } + + #[inline] + fn as_raw(&self) -> *mut bindings::sg_table { + self.inner.0.get() + } + + /// Returns an [`SGTableIter`] bound to the lifetime of `self`. + pub fn iter(&self) -> SGTableIter<'_> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. + let nents = unsafe { (*self.as_raw()).nents }; + + let pos = if nents > 0 { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. + let ptr = unsafe { (*self.as_raw()).sgl }; + + // SAFETY: `ptr` is guaranteed to be a valid pointer to a `struct scatterlist`. + Some(unsafe { SGEntry::from_raw(ptr) }) + } else { + None + }; + + SGTableIter { pos, nents } + } +} + +/// Represents the DMA mapping state of a `struct sg_table`. +/// +/// This is used as an inner type of [`Owned`] to manage the DMA mapping lifecycle. +/// +/// # Invariants +/// +/// - `sgt` is a valid pointer to a `struct sg_table` for the entire lifetime of the +/// [`DmaMappedSgt`]. +/// - `sgt` is always DMA mapped. +struct DmaMappedSgt { + sgt: NonNull, + dev: ARef, + dir: dma::DataDirection, +} + +// SAFETY: `DmaMappedSgt` can be sent to any task. +unsafe impl Send for DmaMappedSgt {} + +// SAFETY: `DmaMappedSgt` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for DmaMappedSgt {} + +impl DmaMappedSgt { + /// # Safety + /// + /// - `sgt` must be a valid pointer to a `struct sg_table` for the entire lifetime of the + /// returned [`DmaMappedSgt`]. + /// - The caller must guarantee that `sgt` remains DMA mapped for the entire lifetime of + /// [`DmaMappedSgt`]. + unsafe fn new( + sgt: NonNull, + dev: &Device, + dir: dma::DataDirection, + ) -> Result { + // SAFETY: + // - `dev.as_raw()` is a valid pointer to a `struct device`, which is guaranteed to be + // bound to a driver for the duration of this call. + // - `sgt` is a valid pointer to a `struct sg_table`. + error::to_result(unsafe { + bindings::dma_map_sgtable(dev.as_raw(), sgt.as_ptr(), dir.into(), 0) + })?; + + // INVARIANT: By the safety requirements of this function it is guaranteed that `sgt` is + // valid for the entire lifetime of this object instance. + Ok(Self { + sgt, + dev: dev.into(), + dir, + }) + } +} + +impl Drop for DmaMappedSgt { + #[inline] + fn drop(&mut self) { + // SAFETY: + // - `self.dev.as_raw()` is a pointer to a valid `struct device`. + // - `self.dev` is the same device the mapping has been created for in `Self::new()`. + // - `self.sgt.as_ptr()` is a valid pointer to a `struct sg_table` by the type invariants + // of `Self`. + // - `self.dir` is the same `dma::DataDirection` the mapping has been created with in + // `Self::new()`. + unsafe { + bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.as_ptr(), self.dir.into(), 0) + }; + } +} + +/// A transparent wrapper around a `struct sg_table`. +/// +/// While we could also create the `struct sg_table` in the constructor of [`Owned`], we can't tear +/// down the `struct sg_table` in [`Owned::drop`]; the drop order in [`Owned`] matters. +#[repr(transparent)] +struct RawSGTable(Opaque); + +// SAFETY: `RawSGTable` can be sent to any task. +unsafe impl Send for RawSGTable {} + +// SAFETY: `RawSGTable` has no interior mutability and can be accessed concurrently. +unsafe impl Sync for RawSGTable {} + +impl RawSGTable { + /// # Safety + /// + /// - `pages` must be a slice of valid `struct page *`. + /// - The pages pointed to by `pages` must remain valid for the entire lifetime of the returned + /// [`RawSGTable`]. + unsafe fn new( + pages: &mut [*mut bindings::page], + size: usize, + max_segment: u32, + flags: alloc::Flags, + ) -> Result { + // `sg_alloc_table_from_pages_segment()` expects at least one page, otherwise it + // produces a NPE. + if pages.is_empty() { + return Err(EINVAL); + } + + let sgt = Opaque::zeroed(); + // SAFETY: + // - `sgt.get()` is a valid pointer to uninitialized memory. + // - As by the check above, `pages` is not empty. + error::to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + pages.as_mut_ptr(), + pages.len().try_into()?, + 0, + size, + max_segment, + flags.as_raw(), + ) + })?; + + Ok(Self(sgt)) + } + + #[inline] + fn as_raw(&self) -> *mut bindings::sg_table { + self.0.get() + } +} + +impl Drop for RawSGTable { + #[inline] + fn drop(&mut self) { + // SAFETY: `sgt` is a valid and initialized `struct sg_table`. + unsafe { bindings::sg_free_table(self.0.get()) }; + } +} + +/// The [`Owned`] generic type of an [`SGTable`]. +/// +/// A [`SGTable`] signifies that the [`SGTable`] owns all associated resources: +/// +/// - The backing memory pages. +/// - The `struct sg_table` allocation (`sgt`). +/// - The DMA mapping, managed through a [`Devres`]-managed `DmaMappedSgt`. +/// +/// Users interact with this type through the [`SGTable`] handle and do not need to manage +/// [`Owned`] directly. +#[pin_data] +pub struct Owned

{ + // Note: The drop order is relevant; we first have to unmap the `struct sg_table`, then free the + // `struct sg_table` and finally free the backing pages. + #[pin] + dma: Devres, + sgt: RawSGTable, + _pages: P, +} + +// SAFETY: `Owned` can be sent to any task if `P` can be send to any task. +unsafe impl Send for Owned

{} + +// SAFETY: `Owned` has no interior mutability and can be accessed concurrently if `P` can be +// accessed concurrently. +unsafe impl Sync for Owned

{} + +impl

Owned

+where + for<'a> P: page::AsPageIter = VmallocPageIter<'a>> + 'static, +{ + fn new( + dev: &Device, + mut pages: P, + dir: dma::DataDirection, + flags: alloc::Flags, + ) -> Result + '_> { + let page_iter = pages.page_iter(); + let size = page_iter.size(); + + let mut page_vec: KVec<*mut bindings::page> = + KVec::with_capacity(page_iter.page_count(), flags)?; + + for page in page_iter { + page_vec.push(page.as_ptr(), flags)?; + } + + // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes + // an `unsigned int`. + // + // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`. + let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { + 0 => u32::MAX, + max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX), + }; + + Ok(try_pin_init!(&this in Self { + // SAFETY: + // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`. + // - The pages contained in `pages` remain valid for the entire lifetime of the + // `RawSGTable`. + sgt: unsafe { RawSGTable::new(&mut page_vec, size, max_segment, flags) }?, + dma <- { + // SAFETY: `this` is a valid pointer to uninitialized memory. + let sgt = unsafe { &raw mut (*this.as_ptr()).sgt }.cast(); + + // SAFETY: `sgt` is guaranteed to be non-null. + let sgt = unsafe { NonNull::new_unchecked(sgt) }; + + // SAFETY: + // - It is guaranteed that the object returned by `DmaMappedSgt::new` won't out-live + // `sgt`. + // - `sgt` is never DMA unmapped manually. + Devres::new(dev, unsafe { DmaMappedSgt::new(sgt, dev, dir) }) + }, + _pages: pages, + })) + } +} + +impl

SGTable> +where + for<'a> P: page::AsPageIter = VmallocPageIter<'a>> + 'static, +{ + /// Allocates a new scatter-gather table from the given pages and maps it for DMA. + /// + /// This constructor creates a new [`SGTable`] that takes ownership of `P`. + /// It allocates a `struct sg_table`, populates it with entries corresponding to the physical + /// pages of `P`, and maps the table for DMA with the specified [`Device`] and + /// [`dma::DataDirection`]. + /// + /// The DMA mapping is managed through [`Devres`], ensuring that the DMA mapping is unmapped + /// once the associated [`Device`] is unbound, or when the [`SGTable`] is dropped. + /// + /// # Parameters + /// + /// * `dev`: The [`Device`] that will be performing the DMA. + /// * `pages`: The entity providing the backing pages. It must implement [`page::AsPageIter`]. + /// The ownership of this entity is moved into the new [`SGTable`]. + /// * `dir`: The [`dma::DataDirection`] of the DMA transfer. + /// * `flags`: Allocation flags for internal allocations (e.g., [`GFP_KERNEL`]). + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// device::{Bound, Device}, + /// dma, page, + /// prelude::*, + /// scatterlist::{SGTable, Owned}, + /// }; + /// + /// fn test(dev: &Device) -> Result { + /// let size = 4 * page::PAGE_SIZE; + /// let pages = VVec::::with_capacity(size, GFP_KERNEL)?; + /// + /// let sgt = KBox::pin_init(SGTable::new( + /// dev, + /// pages, + /// dma::DataDirection::ToDevice, + /// GFP_KERNEL, + /// ), GFP_KERNEL)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn new( + dev: &Device, + pages: P, + dir: dma::DataDirection, + flags: alloc::Flags, + ) -> impl PinInit + '_ { + try_pin_init!(Self { + inner <- Owned::new(dev, pages, dir, flags)? + }) + } +} + +impl

Deref for SGTable> { + type Target = SGTable; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - `self.inner.sgt.as_raw()` is a valid pointer to a `struct sg_table` for the entire + // lifetime of `self`. + // - The backing `struct sg_table` is not modified for the entire lifetime of `self`. + unsafe { SGTable::from_raw(self.inner.sgt.as_raw()) } + } +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::Borrowed {} + impl

Sealed for super::Owned

{} +} + +/// An [`Iterator`] over the DMA mapped [`SGEntry`] items of an [`SGTable`]. +/// +/// Note that the existence of an [`SGTableIter`] does not guarantee that the [`SGEntry`] items +/// actually remain DMA mapped; they are prone to be unmapped on device unbind. +pub struct SGTableIter<'a> { + pos: Option<&'a SGEntry>, + /// The number of DMA mapped entries in a `struct sg_table`. + nents: c_uint, +} + +impl<'a> Iterator for SGTableIter<'a> { + type Item = &'a SGEntry; + + fn next(&mut self) -> Option { + let entry = self.pos?; + self.nents = self.nents.saturating_sub(1); + + // SAFETY: `entry.as_raw()` is a valid pointer to a `struct scatterlist`. + let next = unsafe { bindings::sg_next(entry.as_raw()) }; + + self.pos = (!next.is_null() && self.nents > 0).then(|| { + // SAFETY: If `next` is not NULL, `sg_next()` guarantees to return a valid pointer to + // the next `struct scatterlist`. + unsafe { SGEntry::from_raw(next) } + }); + + Some(entry) + } +} diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs new file mode 100644 index 000000000..9d2716952 --- /dev/null +++ b/rust/kernel/security.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Linux Security Modules (LSM). +//! +//! C header: [`include/linux/security.h`](srctree/include/linux/security.h). + +use crate::{ + bindings, + cred::Credential, + error::{to_result, Result}, + fs::File, +}; + +/// Calls the security modules to determine if the given task can become the manager of a binder +/// context. +#[inline] +pub fn binder_set_context_mgr(mgr: &Credential) -> Result { + // SAFETY: `mrg.0` is valid because the shared reference guarantees a nonzero refcount. + to_result(unsafe { bindings::security_binder_set_context_mgr(mgr.as_ptr()) }) +} + +/// Calls the security modules to determine if binder transactions are allowed from task `from` to +/// task `to`. +#[inline] +pub fn binder_transaction(from: &Credential, to: &Credential) -> Result { + // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts. + to_result(unsafe { bindings::security_binder_transaction(from.as_ptr(), to.as_ptr()) }) +} + +/// Calls the security modules to determine if task `from` is allowed to send binder objects +/// (owned by itself or other processes) to task `to` through a binder transaction. +#[inline] +pub fn binder_transfer_binder(from: &Credential, to: &Credential) -> Result { + // SAFETY: `from` and `to` are valid because the shared references guarantee nonzero refcounts. + to_result(unsafe { bindings::security_binder_transfer_binder(from.as_ptr(), to.as_ptr()) }) +} + +/// Calls the security modules to determine if task `from` is allowed to send the given file to +/// task `to` (which would get its own file descriptor) through a binder transaction. +#[inline] +pub fn binder_transfer_file(from: &Credential, to: &Credential, file: &File) -> Result { + // SAFETY: `from`, `to` and `file` are valid because the shared references guarantee nonzero + // refcounts. + to_result(unsafe { + bindings::security_binder_transfer_file(from.as_ptr(), to.as_ptr(), file.as_ptr()) + }) +} + +/// A security context string. +/// +/// # Invariants +/// +/// The `ctx` field corresponds to a valid security context as returned by a successful call to +/// `security_secid_to_secctx`, that has not yet been released by `security_release_secctx`. +pub struct SecurityCtx { + ctx: bindings::lsm_context, +} + +impl SecurityCtx { + /// Get the security context given its id. + #[inline] + pub fn from_secid(secid: u32) -> Result { + // SAFETY: `struct lsm_context` can be initialized to all zeros. + let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() }; + + // SAFETY: Just a C FFI call. The pointer is valid for writes. + to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut ctx) })?; + + // INVARIANT: If the above call did not fail, then we have a valid security context. + Ok(Self { ctx }) + } + + /// Returns whether the security context is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.ctx.len == 0 + } + + /// Returns the length of this security context. + #[inline] + pub fn len(&self) -> usize { + self.ctx.len as usize + } + + /// Returns the bytes for this security context. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + let ptr = self.ctx.context; + if ptr.is_null() { + debug_assert_eq!(self.len(), 0); + // We can't pass a null pointer to `slice::from_raw_parts` even if the length is zero. + return &[]; + } + + // SAFETY: The call to `security_secid_to_secctx` guarantees that the pointer is valid for + // `self.len()` bytes. Furthermore, if the length is zero, then we have ensured that the + // pointer is not null. + unsafe { core::slice::from_raw_parts(ptr.cast(), self.len()) } + } +} + +impl Drop for SecurityCtx { + #[inline] + fn drop(&mut self) { + // SAFETY: By the invariant of `Self`, this releases an lsm context that came from a + // successful call to `security_secid_to_secctx` and has not yet been released. + unsafe { bindings::security_release_secctx(&mut self.ctx) }; + } +} diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs new file mode 100644 index 000000000..518265558 --- /dev/null +++ b/rust/kernel/seq_file.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Seq file bindings. +//! +//! C header: [`include/linux/seq_file.h`](srctree/include/linux/seq_file.h) + +use crate::{bindings, fmt, str::CStrExt as _, types::NotThreadSafe, types::Opaque}; + +/// A utility for generating the contents of a seq file. +#[repr(transparent)] +pub struct SeqFile { + inner: Opaque, + _not_send: NotThreadSafe, +} + +impl SeqFile { + /// Creates a new [`SeqFile`] from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of `'a` the following is satisfied: + /// * The pointer points at a valid `struct seq_file`. + /// * The `struct seq_file` is not accessed from any other thread. + pub unsafe fn from_raw<'a>(ptr: *mut bindings::seq_file) -> &'a SeqFile { + // SAFETY: The caller ensures that the reference is valid for 'a. There's no way to trigger + // a data race by using the `&SeqFile` since this is the only thread accessing the seq_file. + // + // CAST: The layout of `struct seq_file` and `SeqFile` is compatible. + unsafe { &*ptr.cast() } + } + + /// Used by the [`seq_print`] macro. + #[inline] + pub fn call_printf(&self, args: fmt::Arguments<'_>) { + // SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`. + unsafe { + bindings::seq_printf( + self.inner.get(), + c"%pA".as_char_ptr(), + core::ptr::from_ref(&args).cast::(), + ); + } + } +} + +/// Write to a [`SeqFile`] with the ordinary Rust formatting syntax. +#[macro_export] +macro_rules! seq_print { + ($m:expr, $($arg:tt)+) => ( + $m.call_printf($crate::prelude::fmt!($($arg)+)) + ); +} +pub use seq_print; diff --git a/rust/kernel/serdev.rs b/rust/kernel/serdev.rs new file mode 100644 index 000000000..17ca504b7 --- /dev/null +++ b/rust/kernel/serdev.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the serial device bus. +//! +//! C header: [`include/linux/serdev.h`](srctree/include/linux/serdev.h) + +use crate::{ + acpi, + device, + driver, + error::{ + from_result, + to_result, + VTABLE_DEFAULT_ERROR, // + }, + new_mutex, + of, + prelude::*, + sync::{ + aref::AlwaysRefCounted, + Mutex, // + }, + time::Jiffies, + types::{ + Opaque, + ScopeGuard, // + }, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{offset_of, MaybeUninit}, + ptr::NonNull, // +}; + +/// Parity bit to use with a serial device. +#[repr(u32)] +pub enum Parity { + /// No parity bit. + None = bindings::serdev_parity_SERDEV_PARITY_NONE, + /// Even partiy. + Even = bindings::serdev_parity_SERDEV_PARITY_EVEN, + /// Odd parity. + Odd = bindings::serdev_parity_SERDEV_PARITY_ODD, +} + +/// An adapter for the registration of serial device bus device drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::serdev_device_driver` is a C type declared as `repr(C)`. +// - `PrivateData<'bound, T>` is the type of the driver's device private data. +// - `struct serdev_device_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::serdev_device_driver; + type DriverData<'bound> = PrivateData<'bound, T>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + sdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct serdev_device_driver` on initialization. + unsafe { + (*sdrv.get()).driver.name = name.as_char_ptr(); + (*sdrv.get()).probe = Some(Self::probe_callback); + (*sdrv.get()).remove = Some(Self::remove_callback); + (*sdrv.get()).driver.of_match_table = of_table; + (*sdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::__serdev_device_driver_register(sdrv.get(), module.as_ptr()) }) + } + + unsafe fn unregister(sdrv: &Opaque) { + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::serdev_device_driver_unregister(sdrv.get()) }; + } +} + +#[doc(hidden)] +#[pin_data(PinnedDrop)] +pub struct PrivateData<'bound, T: Driver> { + sdev: &'bound Device, + #[pin] + driver: UnsafeCell>>, + open: UnsafeCell, + /// Whether `receive_buf_callback` is allowed to call `Driver::receive`. + /// + /// If locked, the receive_buf_callback will be blocked on data reception. + /// This is the case while the driver is being probed or while [`PrivateData`] is being dropped. + /// This is necessary, because we need to open the serdev device before the driver has been + /// probed in order to allow it to be configured, which allows `receive_buf_callback` to be + /// called. Thus we need to block data until probe completes and the driver data becomes + /// initialized. + /// + /// If unlocked and true, the receive_buf_callback will forward the data to + /// `Driver::receive`. This is the normal state of operation. + /// + /// If unlocked and false, the receive_buf_callback will throw away the data. + /// This is only the case, if the serdev device is open and + /// - the driver returned an error in probe + /// or + /// - the driver data already has been dropped, because it was unbound. + #[pin] + active: Mutex, +} + +#[pinned_drop] +impl PinnedDrop for PrivateData<'_, T> { + fn drop(self: Pin<&mut Self>) { + let mut active = self.active.lock(); + if *active { + // SAFETY: + // - We have exclusive access to `self.driver`. + // - `self.driver` is guaranteed to be initialized. + unsafe { (*self.driver.get()).assume_init_drop() }; + *active = false; + } + drop(active); + + // SAFETY: We have exclusive access to `self.open`. + if unsafe { *self.open.get() } { + // SAFETY: `self.sdev.as_raw()` is guaranteed to be a pointer to a valid + // `struct serdev_device`. + unsafe { bindings::serdev_device_close(self.sdev.as_raw()) }; + } + } +} + +impl Adapter { + const OPS: &'static bindings::serdev_device_ops = &bindings::serdev_device_ops { + receive_buf: if T::HAS_RECEIVE { + Some(Self::receive_buf_callback) + } else { + None + }, + write_wakeup: Some(bindings::serdev_device_write_wakeup), + }; + + extern "C" fn probe_callback(sdev: *mut bindings::serdev_device) -> kernel::ffi::c_int { + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer to + // a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `probe_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + // SAFETY: `sdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(sdev.as_ref()) }; + + from_result(|| { + sdev.as_ref().set_drvdata(try_pin_init!(PrivateData:: { + sdev: &**sdev, + driver: MaybeUninit::>::zeroed().into(), + open: false.into(), + active <- new_mutex!(false), + }))?; + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let private_data = ScopeGuard::new_with_data(private_data, |_| { + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + drop(unsafe { sdev.as_ref().drvdata_obtain::>() }); + }); + let mut active = private_data.active.lock(); + + // SAFETY: `sdev.as_raw()` is guaranteed to be a valid pointer to `serdev_device`. + unsafe { bindings::serdev_device_set_client_ops(sdev.as_raw(), Self::OPS) }; + + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer + // to a `serdev_device`. + to_result(unsafe { bindings::serdev_device_open(sdev.as_raw()) })?; + + // SAFETY: We have exclusive access to `private_data.open`. + unsafe { *private_data.open.get() = true }; + + let data = T::probe(sdev, info); + + // SAFETY: We have exclusive access to `private_data.driver`. + let driver = unsafe { &mut *private_data.driver.get() }; + // SAFETY: + // - `driver.as_mut_ptr()` is a valid pointer to uninitialized data. + // - `private_data.driver` is pinned. + let result = unsafe { pin_init::raw_try_init(driver.as_mut_ptr(), data) }; + + *active = result.is_ok(); + + drop(active); + + result.map(|()| { + private_data.dismiss(); + 0 + }) + }) + } + + extern "C" fn remove_callback(sdev: *mut bindings::serdev_device) { + // SAFETY: The serial device bus only ever calls the remove callback with a valid pointer + // to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `remove_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `remove_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + T::unbind(sdev, data_pinned); + } + + extern "C" fn receive_buf_callback( + sdev: *mut bindings::serdev_device, + buf: *const u8, + length: usize, + ) -> usize { + // SAFETY: The serial device bus only ever calls the receive buf callback with a valid + // pointer to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `receive_buf_callback()`. + let sdev = unsafe { &*sdev.cast::>() }; + + // SAFETY: `receive_buf_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let active = private_data.active.lock(); + + if !*active { + return length; + } + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `receive_buf_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + // SAFETY: `buf` is guaranteed to be non-null and has the size of `length`. + let buf = unsafe { core::slice::from_raw_parts(buf, length) }; + + T::receive(sdev, data_pinned, buf) + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single serial device bus device driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_serdev_device_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_serdev_device_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::serdev::Adapter, { $($f)* }); + }; +} + +/// The serial device bus device driver trait. +/// +/// Drivers must implement this trait in order to get a serial device bus device driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// acpi, +/// bindings, +/// device::{ +/// Bound, +/// Core, // +/// }, +/// of, +/// serdev, // +/// }; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// #[vtable] +/// impl serdev::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// sdev: &'bound serdev::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// sdev.set_baudrate(115200); +/// sdev.write_all(b"Hello\n", 0)?; +/// Ok(MyDriver) +/// } +/// } +///``` +#[vtable] +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + Sync + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// Serial device bus device driver probe. + /// + /// Called when a new serial device bus device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + sdev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// Serial device bus device driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(sdev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (sdev, this); + } + + /// Serial device bus device data receive callback. + /// + /// Called when data got received from device. + /// + /// Returns the number of bytes accepted. + fn receive<'bound>( + sdev: &'bound Device, + this: Pin<&Self::Data<'bound>>, + data: &[u8], + ) -> usize { + let _ = (sdev, this, data); + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// The serial device bus device representation. +/// +/// This structure represents the Rust abstraction for a C `struct serdev_device`. The +/// implementation abstracts the usage of an already existing C `struct serdev_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct serdev_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::serdev_device { + self.0.get() + } +} + +impl Device { + /// Set the baudrate in bits per second. + /// + /// Common baudrates are 115200, 9600, 19200, 57600, 4800. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_baudrate(&self, speed: u32) -> Result<(), u32> { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + let ret = unsafe { bindings::serdev_device_set_baudrate(self.as_raw(), speed) }; + if ret == speed { + Ok(()) + } else { + Err(ret) + } + } + + /// Set if flow control should be enabled. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_flow_control(&self, enable: bool) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_set_flow_control(self.as_raw(), enable) }; + } + + /// Set parity to use. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_parity(&self, parity: Parity) -> Result { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + to_result(unsafe { bindings::serdev_device_set_parity(self.as_raw(), parity as u32) }) + } + + /// Write data to the serial device until the controller has accepted all the data or has + /// been interrupted by a timeout or signal. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Use a timeout of 0 to wait indefinitely. + /// + /// Returns the number of bytes written (less than `data.len()` if interrupted). + /// [`kernel::error::code::ETIMEDOUT`] or [`kernel::error::code::ERESTARTSYS`] if interrupted + /// before any bytes were written. [`kernel::error::code::EINVAL`] if `data.len() > i32::MAX`. + #[inline] + pub fn write_all(&self, data: &[u8], timeout: Jiffies) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = unsafe { + bindings::serdev_device_write( + self.as_raw(), + data.as_ptr(), + data.len(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + // CAST: negative return values are guaranteed to be between `-MAX_ERRNO` and `-1`, + // which always fit into a `i32`. + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Write data to the serial device. + /// + /// If you want to write until the controller has accepted all the data, use + /// [`Device::write_all`]. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Returns the number of bytes written (less than `data.len()` if not enough room in the + /// write buffer). + #[inline] + pub fn write(&self, data: &[u8]) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = + unsafe { bindings::serdev_device_write_buf(self.as_raw(), data.as_ptr(), data.len()) }; + + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Send data to the serial device immediately. + /// + /// Note that this doesn't guarantee that the data has been transmitted. + /// Use [`Device::wait_until_sent`] for this purpose. + #[inline] + pub fn write_flush(&self) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_write_flush(self.as_raw()) }; + } + + /// Wait for the data to be sent. + /// + /// After this function, the write buffer of the controller should be empty or the timeout + /// elapsed. + /// + /// Use a timeout of 0 to wait indefinitely. + #[inline] + pub fn wait_until_sent(&self, timeout: Jiffies) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { + bindings::serdev_device_wait_until_sent( + self.as_raw(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + } +} + +// SAFETY: `serdev::Device` is a transparent wrapper of `struct serdev_device`. +// The offset is guaranteed to point to a valid device field inside `serdev::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::serdev_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + self.as_ref().inc_ref(); + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::serdev_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct serdev_device`. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct serdev_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs new file mode 100644 index 000000000..521b2b38b --- /dev/null +++ b/rust/kernel/sizes.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Commonly used sizes. +//! +//! C headers: [`include/linux/sizes.h`](srctree/include/linux/sizes.h). +//! +//! The top-level `SZ_*` constants are [`usize`]-typed, for use in kernel page +//! arithmetic and similar CPU-side work. +//! +//! The [`SizeConstants`] trait provides the same constants as associated constants +//! on [`u32`], [`u64`], and [`usize`], for use in device address spaces where +//! the address width depends on the hardware. Device drivers frequently need +//! these constants as [`u64`] (or [`u32`]) rather than [`usize`], because +//! device address spaces are sized independently of the CPU pointer width. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! page::PAGE_SIZE, +//! sizes::{ +//! SizeConstants, +//! SZ_1M, // +//! }, // +//! }; +//! +//! // Module-level constants continue to work without a type qualifier. +//! let num_pages_in_1m = SZ_1M / PAGE_SIZE; +//! +//! // Trait associated constants require a type qualifier. +//! let heap_size = 14 * u64::SZ_1M; +//! let small = u32::SZ_4K; +//! ``` + +macro_rules! define_sizes { + ($($type:ty),* $(,)?) => { + define_sizes!(@internal [$($type),*] + /// `0x0000_0400`. + SZ_1K, + /// `0x0000_0800`. + SZ_2K, + /// `0x0000_1000`. + SZ_4K, + /// `0x0000_2000`. + SZ_8K, + /// `0x0000_4000`. + SZ_16K, + /// `0x0000_8000`. + SZ_32K, + /// `0x0001_0000`. + SZ_64K, + /// `0x0002_0000`. + SZ_128K, + /// `0x0004_0000`. + SZ_256K, + /// `0x0008_0000`. + SZ_512K, + /// `0x0010_0000`. + SZ_1M, + /// `0x0020_0000`. + SZ_2M, + /// `0x0040_0000`. + SZ_4M, + /// `0x0080_0000`. + SZ_8M, + /// `0x0100_0000`. + SZ_16M, + /// `0x0200_0000`. + SZ_32M, + /// `0x0400_0000`. + SZ_64M, + /// `0x0800_0000`. + SZ_128M, + /// `0x1000_0000`. + SZ_256M, + /// `0x2000_0000`. + SZ_512M, + /// `0x4000_0000`. + SZ_1G, + /// `0x8000_0000`. + SZ_2G, + ); + }; + + (@internal [$($type:ty),*] $($names_and_metas:tt)*) => { + define_sizes!(@consts_and_trait $($names_and_metas)*); + define_sizes!(@impls [$($type),*] $($names_and_metas)*); + }; + + (@consts_and_trait $($(#[$meta:meta])* $name:ident,)*) => { + $( + $(#[$meta])* + pub const $name: usize = bindings::$name as usize; + )* + + /// Size constants for device address spaces. + /// + /// Implemented for [`u32`], [`u64`], and [`usize`] so drivers can + /// choose the width that matches their hardware. All `SZ_*` values fit + /// in a [`u32`], so all implementations are lossless. + /// + /// # Examples + /// + /// ``` + /// use kernel::sizes::SizeConstants; + /// + /// let gpu_heap = 14 * u64::SZ_1M; + /// let mmio_window = u32::SZ_16M; + /// ``` + pub trait SizeConstants { + $( + $(#[$meta])* + const $name: Self; + )* + } + }; + + (@impls [] $($(#[$meta:meta])* $name:ident,)*) => {}; + + (@impls [$first:ty $(, $rest:ty)*] $($(#[$meta:meta])* $name:ident,)*) => { + impl SizeConstants for $first { + $( + const $name: Self = { + assert!((self::$name as u128) <= (<$first>::MAX as u128)); + self::$name as $first + }; + )* + } + + define_sizes!(@impls [$($rest),*] $($(#[$meta])* $name,)*); + }; +} + +define_sizes!(u32, u64, usize); diff --git a/rust/kernel/soc.rs b/rust/kernel/soc.rs new file mode 100644 index 000000000..0d6a36c83 --- /dev/null +++ b/rust/kernel/soc.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2025 Google LLC. + +//! SoC Driver Abstraction. +//! +//! C header: [`include/linux/sys_soc.h`](srctree/include/linux/sys_soc.h) + +use crate::{ + bindings, + error, + prelude::*, + str::CString, + types::Opaque, // +}; +use core::ptr::NonNull; + +/// Attributes for a SoC device. +/// +/// These are both exported to userspace under /sys/devices/socX and provided to other drivers to +/// match against via `soc_device_match` (not yet available in Rust) to enable quirks or +/// device-specific support where necessary. +/// +/// All fields are freeform - they have no specific formatting, just defined meanings. +/// For example, the [`machine`](`Attributes::machine`) field could be "DB8500" or +/// "Qualcomm Technologies, Inc. SM8560 HDK", but regardless it should identify a board or product. +pub struct Attributes { + /// Should generally be a board ID or product ID. Examples + /// include DB8500 (ST-Ericsson) or "Qualcomm Technologies, inc. SM8560 HDK". + /// + /// If this field is not populated, the SoC infrastructure will try to populate it from + /// `/model` in the device tree. + pub machine: Option, + /// The broader class this SoC belongs to. Examples include ux500 + /// (for DB8500) or Snapdragon (for SM8650). + /// + /// On chips with ARM firmware supporting SMCCC v1.2+, this may be a JEDEC JEP106 manufacturer + /// identification. + pub family: Option, + /// The manufacturing revision of the part. Frequently this is MAJOR.MINOR, but not always. + pub revision: Option, + /// Serial Number - uniquely identifies a specific SoC. If present, should be unique (buying a + /// replacement part should change it if present). This field cannot be matched on and is + /// solely present to export through /sys. + pub serial_number: Option, + /// SoC ID - identifies a specific SoC kind in question, sometimes more specifically than + /// `machine` if the same SoC is used in multiple products. Some devices use this to specify a + /// SoC name, e.g. "I.MX??", and others just print an ID number (e.g. Tegra and Qualcomm). + /// + /// On chips with ARM firmware supporting SMCCC v1.2+, this may be a JEDEC JEP106 manufacturer + /// identification (the family value) followed by a colon and then a 4-digit ID value. + pub soc_id: Option, +} + +struct BuiltAttributes { + // While `inner` has pointers to `_backing`, it is to the interior of the `CStrings`, not + // `backing` itself, so it does not need to be pinned. + _backing: Attributes, + // `Opaque` makes us `!Unpin`, as the registration holds a pointer to `inner` when used. + inner: Opaque, +} + +fn cstring_to_c(mcs: &Option) -> *const kernel::ffi::c_char { + mcs.as_ref() + .map(|cs| cs.as_char_ptr()) + .unwrap_or(core::ptr::null()) +} + +impl BuiltAttributes { + fn as_mut_ptr(&self) -> *mut bindings::soc_device_attribute { + self.inner.get() + } +} + +impl Attributes { + fn build(self) -> BuiltAttributes { + BuiltAttributes { + inner: Opaque::new(bindings::soc_device_attribute { + machine: cstring_to_c(&self.machine), + family: cstring_to_c(&self.family), + revision: cstring_to_c(&self.revision), + serial_number: cstring_to_c(&self.serial_number), + soc_id: cstring_to_c(&self.soc_id), + data: core::ptr::null(), + custom_attr_group: core::ptr::null(), + }), + _backing: self, + } + } +} + +#[pin_data(PinnedDrop)] +/// Registration handle for your soc_dev. If you let it go out of scope, your soc_dev will be +/// unregistered. +pub struct Registration { + #[pin] + attr: BuiltAttributes, + soc_dev: NonNull, +} + +// SAFETY: We provide no operations through `&Registration`. +unsafe impl Sync for Registration {} + +// SAFETY: All pointers are normal allocations, not thread-specific. +unsafe impl Send for Registration {} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Device always contains a live pointer to a soc_device that can be unregistered + unsafe { bindings::soc_device_unregister(self.soc_dev.as_ptr()) } + } +} + +impl Registration { + /// Register a new SoC device + pub fn new(attr: Attributes) -> impl PinInit { + try_pin_init!(Self { + attr: attr.build(), + soc_dev: { + // SAFETY: + // * The struct provided through attr is backed by pinned data next to it, + // so as long as attr lives, the strings pointed to by the struct will too. + // * `attr` is pinned, so the pinned data won't move. + // * If it returns a device, and so others may try to read this data, by + // caller invariant, `attr` won't be released until the device is. + let raw_soc = error::from_err_ptr(unsafe { + bindings::soc_device_register(attr.as_mut_ptr()) + })?; + + NonNull::new(raw_soc).ok_or(EINVAL)? + }, + }? Error) + } +} diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs new file mode 100644 index 000000000..abbab5050 --- /dev/null +++ b/rust/kernel/std_vendor.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . + +/// [`std::dbg`], but using [`pr_info`] instead of [`eprintln`]. +/// +/// Prints and returns the value of a given expression for quick and dirty +/// debugging. +/// +/// An example: +/// +/// ```rust +/// let a = 2; +/// # #[expect(clippy::disallowed_macros)] +/// let b = dbg!(a * 2) + 1; +/// // ^-- prints: [src/main.rs:3:9] a * 2 = 4 +/// assert_eq!(b, 5); +/// ``` +/// +/// The macro works by using the `Debug` implementation of the type of +/// the given expression to print the value with [`printk`] along with the +/// source location of the macro invocation as well as the source code +/// of the expression. +/// +/// Invoking the macro on an expression moves and takes ownership of it +/// before returning the evaluated expression unchanged. If the type +/// of the expression does not implement `Copy` and you don't want +/// to give up ownership, you can instead borrow with `dbg!(&expr)` +/// for some expression `expr`. +/// +/// The `dbg!` macro works exactly the same in release builds. +/// This is useful when debugging issues that only occur in release +/// builds or when debugging in release mode is significantly faster. +/// +/// Note that the macro is intended as a temporary debugging tool to be +/// used during development. Therefore, avoid committing `dbg!` macro +/// invocations into the kernel tree. +/// +/// For debug output that is intended to be kept in the kernel tree, +/// use [`pr_debug`] and similar facilities instead. +/// +/// # Stability +/// +/// The exact output printed by this macro should not be relied upon +/// and is subject to future changes. +/// +/// # Further examples +/// +/// With a method call: +/// +/// ```rust +/// # #[expect(clippy::disallowed_macros)] +/// fn foo(n: usize) { +/// if dbg!(n.checked_sub(4)).is_some() { +/// // ... +/// } +/// } +/// +/// foo(3) +/// ``` +/// +/// This prints to the kernel log: +/// +/// ```text,ignore +/// [src/main.rs:3:8] n.checked_sub(4) = None +/// ``` +/// +/// Naive factorial implementation: +/// +/// ```rust +/// # #![expect(clippy::disallowed_macros)] +/// fn factorial(n: u32) -> u32 { +/// if dbg!(n <= 1) { +/// dbg!(1) +/// } else { +/// dbg!(n * factorial(n - 1)) +/// } +/// } +/// +/// dbg!(factorial(4)); +/// ``` +/// +/// This prints to the kernel log: +/// +/// ```text,ignore +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = true +/// [src/main.rs:4:9] 1 = 1 +/// [src/main.rs:5:9] n * factorial(n - 1) = 2 +/// [src/main.rs:5:9] n * factorial(n - 1) = 6 +/// [src/main.rs:5:9] n * factorial(n - 1) = 24 +/// [src/main.rs:11:1] factorial(4) = 24 +/// ``` +/// +/// The `dbg!(..)` macro moves the input: +/// +/// ```ignore +/// /// A wrapper around `usize` which importantly is not Copyable. +/// #[derive(Debug)] +/// struct NoCopy(usize); +/// +/// let a = NoCopy(42); +/// let _ = dbg!(a); // <-- `a` is moved here. +/// let _ = dbg!(a); // <-- `a` is moved again; error! +/// ``` +/// +/// You can also use `dbg!()` without a value to just print the +/// file and line whenever it's reached. +/// +/// Finally, if you want to `dbg!(..)` multiple values, it will treat them as +/// a tuple (and return it, too): +/// +/// ``` +/// # #![expect(clippy::disallowed_macros)] +/// assert_eq!(dbg!(1usize, 2u32), (1, 2)); +/// ``` +/// +/// However, a single argument with a trailing comma will still not be treated +/// as a tuple, following the convention of ignoring trailing commas in macro +/// invocations. You can use a 1-tuple directly if you need one: +/// +/// ``` +/// # #![expect(clippy::disallowed_macros)] +/// assert_eq!(1, dbg!(1u32,)); // trailing comma ignored +/// assert_eq!((1,), dbg!((1u32,))); // 1-tuple +/// ``` +/// +/// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html +/// [`eprintln`]: https://doc.rust-lang.org/std/macro.eprintln.html +/// [`printk`]: https://docs.kernel.org/core-api/printk-basics.html +/// [`pr_info`]: crate::pr_info! +/// [`pr_debug`]: crate::pr_debug! +#[macro_export] +macro_rules! dbg { + // NOTE: We cannot use `concat!` to make a static string as a format argument + // of `pr_info!` because `file!` could contain a `{` or + // `$val` expression could be a block (`{ .. }`), in which case the `pr_info!` + // will be malformed. + () => { + $crate::pr_info!("[{}:{}:{}]\n", ::core::file!(), ::core::line!(), ::core::column!()) + }; + ($val:expr $(,)?) => { + // Use of `match` here is intentional because it affects the lifetimes + // of temporaries - + match $val { + tmp => { + $crate::pr_info!("[{}:{}:{}] {} = {:#?}\n", + ::core::file!(), ::core::line!(), ::core::column!(), + ::core::stringify!($val), &tmp); + tmp + } + } + }; + ($($val:expr),+ $(,)?) => { + ($($crate::dbg!($val)),+,) + }; +} diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs new file mode 100644 index 000000000..b3caa9a1c --- /dev/null +++ b/rust/kernel/str.rs @@ -0,0 +1,918 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! String representations. + +use crate::{ + alloc::{ + AllocError, + KVec, // + }, + error::{ + to_result, + Result, // + }, + fmt::{ + self, + Write, // + }, + prelude::*, // +}; +use core::{ + marker::PhantomData, + ops::{ + Deref, + DerefMut, + Index, // + }, // +}; + +pub use crate::prelude::CStr; + +pub mod parse_int; + +/// Byte string without UTF-8 validity guarantee. +#[repr(transparent)] +pub struct BStr([u8]); + +impl BStr { + /// Returns the length of this string. + #[inline] + pub const fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the string is empty. + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a [`BStr`] from a `[u8]`. + #[inline] + pub const fn from_bytes(bytes: &[u8]) -> &Self { + // SAFETY: `BStr` is transparent to `[u8]`. + unsafe { &*(core::ptr::from_ref(bytes) as *const BStr) } + } + + /// Strip a prefix from `self`. Delegates to [`slice::strip_prefix`]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::b_str; + /// assert_eq!(Some(b_str!("bar")), b_str!("foobar").strip_prefix(b_str!("foo"))); + /// assert_eq!(None, b_str!("foobar").strip_prefix(b_str!("bar"))); + /// assert_eq!(Some(b_str!("foobar")), b_str!("foobar").strip_prefix(b_str!(""))); + /// assert_eq!(Some(b_str!("")), b_str!("foobar").strip_prefix(b_str!("foobar"))); + /// ``` + pub fn strip_prefix(&self, pattern: impl AsRef) -> Option<&BStr> { + self.deref() + .strip_prefix(pattern.as_ref().deref()) + .map(Self::from_bytes) + } +} + +impl fmt::Display for BStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; + /// let ascii = b_str!("Hello, BStr!"); + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes(), "Hello, BStr!".as_bytes()); + /// + /// let non_ascii = b_str!("🦀"); + /// let s = CString::try_from_fmt(fmt!("{non_ascii}"))?; + /// assert_eq!(s.to_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{b:02x}")?, + } + } + Ok(()) + } +} + +impl fmt::Debug for BStr { + /// Formats printable ASCII characters with a double quote on either end, + /// escaping the rest. + /// + /// ``` + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; + /// // Embedded double quotes are escaped. + /// let ascii = b_str!("Hello, \"BStr\"!"); + /// let s = CString::try_from_fmt(fmt!("{ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); + /// + /// let non_ascii = b_str!("😺"); + /// let s = CString::try_from_fmt(fmt!("{non_ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // String escape characters. + b'\"' => f.write_str("\\\"")?, + b'\\' => f.write_str("\\\\")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{b:02x}")?, + } + } + f.write_char('"') + } +} + +impl Deref for BStr { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for BStr { + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl Index for BStr +where + [u8]: Index, +{ + type Output = Self; + + fn index(&self, index: Idx) -> &Self::Output { + BStr::from_bytes(&self.0[index]) + } +} + +impl AsRef for [u8] { + fn as_ref(&self) -> &BStr { + BStr::from_bytes(self) + } +} + +impl AsRef for BStr { + fn as_ref(&self) -> &BStr { + self + } +} + +/// Creates a new [`BStr`] from a string literal. +/// +/// `b_str!` converts the supplied string literal to byte string, so non-ASCII +/// characters can be included. +/// +/// # Examples +/// +/// ``` +/// # use kernel::b_str; +/// # use kernel::str::BStr; +/// const MY_BSTR: &BStr = b_str!("My awesome BStr!"); +/// ``` +#[macro_export] +macro_rules! b_str { + ($str:literal) => {{ + const S: &'static str = $str; + const C: &'static $crate::str::BStr = $crate::str::BStr::from_bytes(S.as_bytes()); + C + }}; +} + +/// Returns a C pointer to the string. +// It is a free function rather than a method on an extension trait because: +// +// - error[E0379]: functions in trait impls cannot be declared const +#[inline] +#[expect(clippy::disallowed_methods, reason = "internal implementation")] +pub const fn as_char_ptr_in_const_context(c_str: &CStr) -> *const c_char { + c_str.as_ptr().cast() +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::CStr {} +} + +/// Extensions to [`CStr`]. +pub trait CStrExt: private::Sealed { + /// Wraps a raw C string pointer. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a `NUL`-terminated C string, and it must + /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr` + /// must not be mutated. + // This function exists to paper over the fact that `CStr::from_ptr` takes a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self; + + /// Creates a mutable [`CStr`] from a `[u8]` without performing any + /// additional checks. + /// + /// # Safety + /// + /// `bytes` *must* end with a `NUL` byte, and should only have a single + /// `NUL` byte (or the string will be truncated). + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self; + + /// Returns a C pointer to the string. + // This function exists to paper over the fact that `CStr::as_ptr` returns a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + fn as_char_ptr(&self) -> *const c_char; + + /// Convert this [`CStr`] into a [`CString`] by allocating memory and + /// copying over the string data. + fn to_cstring(&self) -> Result; + + /// Converts this [`CStr`] to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`to_ascii_lowercase()`]. + /// + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase + fn make_ascii_lowercase(&mut self); + + /// Converts this [`CStr`] to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`to_ascii_uppercase()`]. + /// + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase + fn make_ascii_uppercase(&mut self); + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// + /// [`make_ascii_lowercase`]: str::make_ascii_lowercase + fn to_ascii_lowercase(&self) -> Result; + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// + /// [`make_ascii_uppercase`]: str::make_ascii_uppercase + fn to_ascii_uppercase(&self) -> Result; +} + +impl fmt::Display for CStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::prelude::fmt; + /// # use kernel::str::CStr; + /// # use kernel::str::CString; + /// let penguin = c"🐧"; + /// let s = CString::try_from_fmt(fmt!("{penguin}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); + /// + /// let ascii = c"so \"cool\""; + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "so \"cool\"\0".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &c in self.to_bytes() { + if (0x20..0x7f).contains(&c) { + // Printable character. + f.write_char(c as char)?; + } else { + write!(f, "\\x{c:02x}")?; + } + } + Ok(()) + } +} + +/// Converts a mutable C string to a mutable byte slice. +/// +/// # Safety +/// +/// The caller must ensure that the slice ends in a NUL byte and contains no other NUL bytes before +/// the borrow ends and the underlying [`CStr`] is used. +unsafe fn to_bytes_mut(s: &mut CStr) -> &mut [u8] { + // SAFETY: the cast from `&CStr` to `&[u8]` is safe since `CStr` has the same layout as `&[u8]` + // (this is technically not guaranteed, but we rely on it here). The pointer dereference is + // safe since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(s) as *mut [u8]) } +} + +impl CStrExt for CStr { + #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { + // SAFETY: The safety preconditions are the same as for `CStr::from_ptr`. + unsafe { CStr::from_ptr(ptr.cast()) } + } + + #[inline] + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self { + // SAFETY: the cast from `&[u8]` to `&CStr` is safe since the properties of `bytes` are + // guaranteed by the safety precondition and `CStr` has the same layout as `&[u8]` (this is + // technically not guaranteed, but we rely on it here). The pointer dereference is safe + // since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(bytes) as *mut CStr) } + } + + #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] + fn as_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } + + fn to_cstring(&self) -> Result { + CString::try_from(self) + } + + fn make_ascii_lowercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_lowercase(); + } + + fn make_ascii_uppercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_uppercase(); + } + + fn to_ascii_lowercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_lowercase(); + + Ok(s) + } + + fn to_ascii_uppercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_uppercase(); + + Ok(s) + } +} + +impl AsRef for CStr { + #[inline] + fn as_ref(&self) -> &BStr { + BStr::from_bytes(self.to_bytes()) + } +} + +/// Creates a new [`CStr`] at compile time. +/// +/// Rust supports C string literals since Rust 1.77, and they should be used instead of this macro +/// where possible. This macro exists to allow static *non-literal* C strings to be created at +/// compile time. This is most often used in other macros. +/// +/// # Panics +/// +/// This macro panics if the operand contains an interior `NUL` byte. +/// +/// # Examples +/// +/// ``` +/// # use kernel::c_str; +/// # use kernel::str::CStr; +/// // This is allowed, but `c"literal"` should be preferred for literals. +/// const BAD: &CStr = c_str!("literal"); +/// +/// // `c_str!` is still needed for static non-literal C strings. +/// const GOOD: &CStr = c_str!(concat!(file!(), ":", line!(), ": My CStr!")); +/// ``` +#[macro_export] +macro_rules! c_str { + // NB: We could write `($str:lit) => compile_error!("use a C string literal instead");` here but + // that would trigger when the literal is at the top of several macro expansions. That would be + // too limiting to macro authors. + ($str:expr) => {{ + const S: &str = concat!($str, "\0"); + const C: &$crate::str::CStr = match $crate::str::CStr::from_bytes_with_nul(S.as_bytes()) { + Ok(v) => v, + Err(_) => panic!("string contains interior NUL"), + }; + C + }}; +} + +#[cfg(CONFIG_RUST_STR_KUNIT_TEST)] +#[kunit_tests(rust_kernel_str)] +mod tests { + use super::*; + + impl From for Error { + #[inline] + fn from(_: core::ffi::FromBytesWithNulError) -> Error { + EINVAL + } + } + + macro_rules! format { + ($($f:tt)*) => ({ + CString::try_from_fmt(fmt!($($f)*))?.to_str()? + }) + } + + const ALL_ASCII_CHARS: &str = + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ + \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \ + !\"#$%&'()*+,-./0123456789:;<=>?@\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\ + \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\ + \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\ + \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\ + \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\ + \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\ + \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\ + \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\ + \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff"; + + #[test] + fn test_cstr_to_str() -> Result { + let cstr = c"\xf0\x9f\xa6\x80"; + let checked_str = cstr.to_str()?; + assert_eq!(checked_str, "🦀"); + Ok(()) + } + + #[test] + fn test_cstr_to_str_invalid_utf8() -> Result { + let cstr = c"\xc3\x28"; + assert!(cstr.to_str().is_err()); + Ok(()) + } + + #[test] + fn test_cstr_display() -> Result { + let hello_world = c"hello, world!"; + assert_eq!(format!("{hello_world}"), "hello, world!"); + let non_printables = c"\x01\x09\x0a"; + assert_eq!(format!("{non_printables}"), "\\x01\\x09\\x0a"); + let non_ascii = c"d\xe9j\xe0 vu"; + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); + let good_bytes = c"\xf0\x9f\xa6\x80"; + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); + Ok(()) + } + + #[test] + fn test_cstr_display_all_bytes() -> Result { + let mut bytes: [u8; 256] = [0; 256]; + // fill `bytes` with [1..=255] + [0] + for i in u8::MIN..=u8::MAX { + bytes[i as usize] = i.wrapping_add(1); + } + let cstr = CStr::from_bytes_with_nul(&bytes)?; + assert_eq!(format!("{cstr}"), ALL_ASCII_CHARS); + Ok(()) + } + + #[test] + fn test_cstr_debug() -> Result { + let hello_world = c"hello, world!"; + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); + let non_printables = c"\x01\x09\x0a"; + assert_eq!(format!("{non_printables:?}"), "\"\\x01\\t\\n\""); + let non_ascii = c"d\xe9j\xe0 vu"; + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); + Ok(()) + } + + #[test] + fn test_bstr_display() -> Result { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{hello_world}"), "hello, world!"); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{escapes}"), "_\\t_\\n_\\r_\\_'_\"_"); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{others}"), "\\x01"); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); + Ok(()) + } + + #[test] + fn test_bstr_debug() -> Result { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{escapes:?}"), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{others:?}"), "\"\\x01\""); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\""); + Ok(()) + } +} + +/// Allows formatting of [`fmt::Arguments`] into a raw buffer. +/// +/// It does not fail if callers write past the end of the buffer so that they can calculate the +/// size required to fit everything. +/// +/// # Invariants +/// +/// The memory region between `pos` (inclusive) and `end` (exclusive) is valid for writes if `pos` +/// is less than `end`. +pub struct RawFormatter { + // Use `usize` to use `saturating_*` functions. + beg: usize, + pos: usize, + end: usize, +} + +impl RawFormatter { + /// Creates a new instance of [`RawFormatter`] with an empty buffer. + fn new() -> Self { + // INVARIANT: The buffer is empty, so the region that needs to be writable is empty. + Self { + beg: 0, + pos: 0, + end: 0, + } + } + + /// Creates a new instance of [`RawFormatter`] with the given buffer pointers. + /// + /// # Safety + /// + /// If `pos` is less than `end`, then the region between `pos` (inclusive) and `end` + /// (exclusive) must be valid for writes for the lifetime of the returned [`RawFormatter`]. + pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self { + // INVARIANT: The safety requirements guarantee the type invariants. + Self { + beg: pos as usize, + pos: pos as usize, + end: end as usize, + } + } + + /// Creates a new instance of [`RawFormatter`] with the given buffer. + /// + /// # Safety + /// + /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes + /// for the lifetime of the returned [`RawFormatter`]. + pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self { + let pos = buf as usize; + // INVARIANT: We ensure that `end` is never less than `buf`, and the safety requirements + // guarantees that the memory region is valid for writes. + Self { + pos, + beg: pos, + end: pos.saturating_add(len), + } + } + + /// Returns the current insert position. + /// + /// N.B. It may point to invalid memory. + pub(crate) fn pos(&self) -> *mut u8 { + self.pos as *mut u8 + } + + /// Returns the number of bytes written to the formatter. + pub fn bytes_written(&self) -> usize { + self.pos - self.beg + } +} + +impl fmt::Write for RawFormatter { + fn write_str(&mut self, s: &str) -> fmt::Result { + // `pos` value after writing `len` bytes. This does not have to be bounded by `end`, but we + // don't want it to wrap around to 0. + let pos_new = self.pos.saturating_add(s.len()); + + // Amount that we can copy. `saturating_sub` ensures we get 0 if `pos` goes past `end`. + let len_to_copy = core::cmp::min(pos_new, self.end).saturating_sub(self.pos); + + if len_to_copy > 0 { + // SAFETY: If `len_to_copy` is non-zero, then we know `pos` has not gone past `end` + // yet, so it is valid for write per the type invariants. + unsafe { + core::ptr::copy_nonoverlapping( + s.as_bytes().as_ptr(), + self.pos as *mut u8, + len_to_copy, + ) + }; + } + + self.pos = pos_new; + Ok(()) + } +} + +/// Allows formatting of [`fmt::Arguments`] into a raw buffer. +/// +/// Fails if callers attempt to write more than will fit in the buffer. +pub struct Formatter<'a>(RawFormatter, PhantomData<&'a mut ()>); + +impl Formatter<'_> { + /// Creates a new instance of [`Formatter`] with the given buffer. + /// + /// # Safety + /// + /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes + /// for the lifetime of the returned [`Formatter`]. + pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self { + // SAFETY: The safety requirements of this function satisfy those of the callee. + Self(unsafe { RawFormatter::from_buffer(buf, len) }, PhantomData) + } + + /// Create a new [`Self`] instance. + pub fn new(buffer: &mut [u8]) -> Self { + // SAFETY: `buffer` is valid for writes for the entire length for + // the lifetime of `Self`. + unsafe { Formatter::from_buffer(buffer.as_mut_ptr(), buffer.len()) } + } +} + +impl Deref for Formatter<'_> { + type Target = RawFormatter; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Write for Formatter<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.0.write_str(s)?; + + // Fail the request if we go past the end of the buffer. + if self.0.pos > self.0.end { + Err(fmt::Error) + } else { + Ok(()) + } + } +} + +/// A mutable reference to a byte buffer where a string can be written into. +/// +/// The buffer will be automatically null terminated after the last written character. +/// +/// # Invariants +/// +/// * The first byte of `buffer` is always zero. +/// * The length of `buffer` is at least 1. +pub struct NullTerminatedFormatter<'a> { + buffer: &'a mut [u8], +} + +impl<'a> NullTerminatedFormatter<'a> { + /// Create a new [`Self`] instance. + pub fn new(buffer: &'a mut [u8]) -> Option> { + *(buffer.first_mut()?) = 0; + + // INVARIANT: + // - We wrote zero to the first byte above. + // - If buffer was not at least length 1, `buffer.first_mut()` would return None. + Some(Self { buffer }) + } +} + +impl Write for NullTerminatedFormatter<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + let len = bytes.len(); + + // We want space for a zero. By type invariant, buffer length is always at least 1, so no + // underflow. + if len > self.buffer.len() - 1 { + return Err(fmt::Error); + } + + let buffer = core::mem::take(&mut self.buffer); + // We break the zero start invariant for a short while. + buffer[..len].copy_from_slice(bytes); + // INVARIANT: We checked above that buffer will have size at least 1 after this assignment. + self.buffer = &mut buffer[len..]; + + // INVARIANT: We write zero to the first byte of the buffer. + self.buffer[0] = 0; + + Ok(()) + } +} + +/// # Safety +/// +/// - `string` must point to a null terminated string that is valid for read. +unsafe fn kstrtobool_raw(string: *const u8) -> Result { + let mut result: bool = false; + + // SAFETY: + // - By function safety requirement, `string` is a valid null-terminated string. + // - `result` is a valid `bool` that we own. + to_result(unsafe { bindings::kstrtobool(string, &mut result) })?; + Ok(result) +} + +/// Convert common user inputs into boolean values using the kernel's `kstrtobool` function. +/// +/// This routine returns `Ok(bool)` if the first character is one of 'YyTt1NnFf0', or +/// \[oO\]\[NnFf\] for "on" and "off". Otherwise it will return `Err(EINVAL)`. +/// +/// # Examples +/// +/// ``` +/// # use kernel::str::kstrtobool; +/// +/// // Lowercase +/// assert_eq!(kstrtobool(c"true"), Ok(true)); +/// assert_eq!(kstrtobool(c"tr"), Ok(true)); +/// assert_eq!(kstrtobool(c"t"), Ok(true)); +/// assert_eq!(kstrtobool(c"twrong"), Ok(true)); +/// assert_eq!(kstrtobool(c"false"), Ok(false)); +/// assert_eq!(kstrtobool(c"f"), Ok(false)); +/// assert_eq!(kstrtobool(c"yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"no"), Ok(false)); +/// assert_eq!(kstrtobool(c"on"), Ok(true)); +/// assert_eq!(kstrtobool(c"off"), Ok(false)); +/// +/// // Camel case +/// assert_eq!(kstrtobool(c"True"), Ok(true)); +/// assert_eq!(kstrtobool(c"False"), Ok(false)); +/// assert_eq!(kstrtobool(c"Yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"No"), Ok(false)); +/// assert_eq!(kstrtobool(c"On"), Ok(true)); +/// assert_eq!(kstrtobool(c"Off"), Ok(false)); +/// +/// // All caps +/// assert_eq!(kstrtobool(c"TRUE"), Ok(true)); +/// assert_eq!(kstrtobool(c"FALSE"), Ok(false)); +/// assert_eq!(kstrtobool(c"YES"), Ok(true)); +/// assert_eq!(kstrtobool(c"NO"), Ok(false)); +/// assert_eq!(kstrtobool(c"ON"), Ok(true)); +/// assert_eq!(kstrtobool(c"OFF"), Ok(false)); +/// +/// // Numeric +/// assert_eq!(kstrtobool(c"1"), Ok(true)); +/// assert_eq!(kstrtobool(c"0"), Ok(false)); +/// +/// // Invalid input +/// assert_eq!(kstrtobool(c"invalid"), Err(EINVAL)); +/// assert_eq!(kstrtobool(c"2"), Err(EINVAL)); +/// ``` +pub fn kstrtobool(string: &CStr) -> Result { + // SAFETY: + // - The pointer returned by `CStr::as_char_ptr` is guaranteed to be + // null terminated. + // - `string` is live and thus the string is valid for read. + unsafe { kstrtobool_raw(string.as_char_ptr()) } +} + +/// Convert `&[u8]` to `bool` by deferring to [`kernel::str::kstrtobool`]. +/// +/// Only considers at most the first two bytes of `bytes`. +pub fn kstrtobool_bytes(bytes: &[u8]) -> Result { + // `ktostrbool` only considers the first two bytes of the input. + let stack_string = [*bytes.first().unwrap_or(&0), *bytes.get(1).unwrap_or(&0), 0]; + // SAFETY: `stack_string` is null terminated and it is live on the stack so + // it is valid for read. + unsafe { kstrtobool_raw(stack_string.as_ptr()) } +} + +/// An owned string that is guaranteed to have exactly one `NUL` byte, which is at the end. +/// +/// Used for interoperability with kernel APIs that take C strings. +/// +/// # Invariants +/// +/// The string is always `NUL`-terminated and contains no other `NUL` bytes. +/// +/// # Examples +/// +/// ``` +/// use kernel::{str::CString, prelude::fmt}; +/// +/// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20))?; +/// assert_eq!(s.to_bytes_with_nul(), "abc1020\0".as_bytes()); +/// +/// let tmp = "testing"; +/// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123))?; +/// assert_eq!(s.to_bytes_with_nul(), "testing123\0".as_bytes()); +/// +/// // This fails because it has an embedded `NUL` byte. +/// let s = CString::try_from_fmt(fmt!("a\0b{}", 123)); +/// assert_eq!(s.is_ok(), false); +/// # Ok::<(), kernel::error::Error>(()) +/// ``` +pub struct CString { + buf: KVec, +} + +impl CString { + /// Creates an instance of [`CString`] from the given formatted arguments. + pub fn try_from_fmt(args: fmt::Arguments<'_>) -> Result { + // Calculate the size needed (formatted string plus `NUL` terminator). + let mut f = RawFormatter::new(); + f.write_fmt(args)?; + f.write_str("\0")?; + let size = f.bytes_written(); + + // Allocate a vector with the required number of bytes, and write to it. + let mut buf = KVec::with_capacity(size, GFP_KERNEL)?; + // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. + let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; + f.write_fmt(args)?; + f.write_str("\0")?; + + // SAFETY: The number of bytes that can be written to `f` is bounded by `size`, which is + // `buf`'s capacity. The `Formatter` is created with `size` as its limit, and the `?` + // operators on `write_fmt` and `write_str` above ensure that if writing exceeds this + // limit, an error is returned early. The contents of the buffer have been initialised + // by writes to `f`. + unsafe { buf.inc_len(f.bytes_written()) }; + + // Check that there are no `NUL` bytes before the end. + // SAFETY: The buffer is valid for read because `f.bytes_written()` is bounded by `size` + // (which the minimum buffer size) and is non-zero (we wrote at least the `NUL` terminator) + // so `f.bytes_written() - 1` doesn't underflow. + let ptr = unsafe { bindings::memchr(buf.as_ptr().cast(), 0, f.bytes_written() - 1) }; + if !ptr.is_null() { + return Err(EINVAL); + } + + // INVARIANT: We wrote the `NUL` terminator and checked above that no other `NUL` bytes + // exist in the buffer. + Ok(Self { buf }) + } +} + +impl Deref for CString { + type Target = CStr; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the string is `NUL`-terminated and that no + // other `NUL` bytes exist. + unsafe { CStr::from_bytes_with_nul_unchecked(self.buf.as_slice()) } + } +} + +impl DerefMut for CString { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: A `CString` is always NUL-terminated and contains no other + // NUL bytes. + unsafe { CStr::from_bytes_with_nul_unchecked_mut(self.buf.as_mut_slice()) } + } +} + +impl<'a> TryFrom<&'a CStr> for CString { + type Error = AllocError; + + fn try_from(cstr: &'a CStr) -> Result { + let mut buf = KVec::new(); + + buf.extend_from_slice(cstr.to_bytes_with_nul(), GFP_KERNEL)?; + + // INVARIANT: The `CStr` and `CString` types have the same invariants for + // the string data, and we copied it over without changes. + Ok(CString { buf }) + } +} + +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} diff --git a/rust/kernel/str/parse_int.rs b/rust/kernel/str/parse_int.rs new file mode 100644 index 000000000..48eb4c202 --- /dev/null +++ b/rust/kernel/str/parse_int.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Integer parsing functions. +//! +//! Integer parsing functions for parsing signed and unsigned integers +//! potentially prefixed with `0x`, `0o`, or `0b`. + +use crate::prelude::*; +use crate::str::BStr; +use core::ops::Deref; + +// Make `FromStrRadix` a public type with a private name. This seals +// `ParseInt`, that is, prevents downstream users from implementing the +// trait. +mod private { + use crate::prelude::*; + use crate::str::BStr; + + /// Trait that allows parsing a [`&BStr`] to an integer with a radix. + pub trait FromStrRadix: Sized { + /// Parse `src` to [`Self`] using radix `radix`. + fn from_str_radix(src: &BStr, radix: u32) -> Result; + + /// Tries to convert `value` into [`Self`] and negates the resulting value. + fn from_u64_negated(value: u64) -> Result; + } +} + +/// Extract the radix from an integer literal optionally prefixed with +/// one of `0x`, `0X`, `0o`, `0O`, `0b`, `0B`, `0`. +fn strip_radix(src: &BStr) -> (u32, &BStr) { + match src.deref() { + [b'0', b'x' | b'X', rest @ ..] => (16, rest.as_ref()), + [b'0', b'o' | b'O', rest @ ..] => (8, rest.as_ref()), + [b'0', b'b' | b'B', rest @ ..] => (2, rest.as_ref()), + // NOTE: We are including the leading zero to be able to parse + // literal `0` here. If we removed it as a radix prefix, we would + // not be able to parse `0`. + [b'0', ..] => (8, src), + _ => (10, src), + } +} + +/// Trait for parsing string representations of integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtoul +/// +/// # Examples +/// +/// ``` +/// # use kernel::str::parse_int::ParseInt; +/// # use kernel::b_str; +/// +/// assert_eq!(Ok(0u8), u8::from_str(b_str!("0"))); +/// +/// assert_eq!(Ok(0xa2u8), u8::from_str(b_str!("0xa2"))); +/// assert_eq!(Ok(-0xa2i32), i32::from_str(b_str!("-0xa2"))); +/// +/// assert_eq!(Ok(-0o57i8), i8::from_str(b_str!("-0o57"))); +/// assert_eq!(Ok(0o57i8), i8::from_str(b_str!("057"))); +/// +/// assert_eq!(Ok(0b1001i16), i16::from_str(b_str!("0b1001"))); +/// assert_eq!(Ok(-0b1001i16), i16::from_str(b_str!("-0b1001"))); +/// +/// assert_eq!(Ok(127i8), i8::from_str(b_str!("127"))); +/// assert!(i8::from_str(b_str!("128")).is_err()); +/// assert_eq!(Ok(-128i8), i8::from_str(b_str!("-128"))); +/// assert!(i8::from_str(b_str!("-129")).is_err()); +/// assert_eq!(Ok(255u8), u8::from_str(b_str!("255"))); +/// assert!(u8::from_str(b_str!("256")).is_err()); +/// ``` +pub trait ParseInt: private::FromStrRadix + TryFrom { + /// Parse a string according to the description in [`Self`]. + fn from_str(src: &BStr) -> Result { + match src.deref() { + [b'-', rest @ ..] => { + let (radix, digits) = strip_radix(rest.as_ref()); + // 2's complement values range from -2^(b-1) to 2^(b-1)-1. + // So if we want to parse negative numbers as positive and + // later multiply by -1, we have to parse into a larger + // integer. We choose `u64` as sufficiently large. + // + // NOTE: 128 bit integers are not available on all + // platforms, hence the choice of 64 bits. + let val = + u64::from_str_radix(core::str::from_utf8(digits).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL)?; + Self::from_u64_negated(val) + } + _ => { + let (radix, digits) = strip_radix(src); + Self::from_str_radix(digits, radix).map_err(|_| EINVAL) + } + } + } +} + +macro_rules! impl_parse_int { + ($($ty:ty),*) => { + $( + impl private::FromStrRadix for $ty { + fn from_str_radix(src: &BStr, radix: u32) -> Result { + <$ty>::from_str_radix(core::str::from_utf8(src).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL) + } + + fn from_u64_negated(value: u64) -> Result { + const ABS_MIN: u64 = { + #[allow(unused_comparisons)] + if <$ty>::MIN < 0 { + 1u64 << (<$ty>::BITS - 1) + } else { + 0 + } + }; + + if value > ABS_MIN { + return Err(EINVAL); + } + + if value == ABS_MIN { + return Ok(<$ty>::MIN); + } + + // SAFETY: The above checks guarantee that `value` fits into `Self`: + // - if `Self` is unsigned, then `ABS_MIN == 0` and thus we have returned above + // (either `EINVAL` or `MIN`). + // - if `Self` is signed, then we have that `0 <= value < ABS_MIN`. And since + // `ABS_MIN - 1` fits into `Self` by construction, `value` also does. + let value: Self = unsafe { value.try_into().unwrap_unchecked() }; + + Ok((!value).wrapping_add(1)) + } + } + + impl ParseInt for $ty {} + )* + }; +} + +impl_parse_int![i8, u8, i16, u16, i32, u32, i64, u64, isize, usize]; diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs new file mode 100644 index 000000000..3e55de3b1 --- /dev/null +++ b/rust/kernel/sync.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Synchronisation primitives. +//! +//! This module contains the kernel APIs related to synchronisation that have been ported or +//! wrapped for usage by Rust code in the kernel. + +use crate::prelude::*; +use crate::types::Opaque; +use pin_init; + +mod arc; +pub mod aref; +pub mod atomic; +pub mod barrier; +pub mod completion; +mod condvar; +pub mod lock; +mod locked_by; +pub mod poll; +pub mod rcu; +mod refcount; +mod set_once; +pub mod srcu; + +pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use completion::Completion; +pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; +pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; +pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; +pub use locked_by::LockedBy; +pub use refcount::Refcount; +pub use set_once::SetOnce; +pub use srcu::Srcu; + +/// Represents a lockdep class. +/// +/// Wraps the kernel's `struct lock_class_key`. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct LockClassKey { + #[pin] + inner: Opaque, +} + +// SAFETY: Unregistering a lock class key from a different thread than where it was registered is +// allowed. +unsafe impl Send for LockClassKey {} + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + +impl LockClassKey { + /// Initializes a statically allocated lock class key. + /// + /// This is usually used indirectly through the [`static_lock_class!`] macro. See its + /// documentation for more information. + /// + /// # Safety + /// + /// * Before using the returned value, it must be pinned in a static memory location. + /// * The destructor must never run on the returned `LockClassKey`. + pub const unsafe fn new_static() -> Self { + LockClassKey { + inner: Opaque::uninit(), + } + } + + /// Initializes a dynamically allocated lock class key. + /// + /// In the common case of using a statically allocated lock class key, the + /// [`static_lock_class!`] macro should be used instead. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::KBox; + /// use kernel::types::ForeignOwnable; + /// use kernel::sync::{LockClassKey, SpinLock}; + /// use pin_init::stack_pin_init; + /// + /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; + /// let key_ptr = key.into_foreign(); + /// + /// { + /// stack_pin_init!(let num: SpinLock = SpinLock::new( + /// 0, + /// c"my_spinlock", + /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose + /// // `from_foreign()` has not yet been called. + /// unsafe { > as ForeignOwnable>::borrow(key_ptr) } + /// )); + /// } + /// + /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous + /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. + /// unsafe { drop(> as ForeignOwnable>::from_foreign(key_ptr)) }; + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_dynamic() -> impl PinInit { + pin_init!(Self { + // SAFETY: lockdep_register_key expects an uninitialized block of memory + inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) + }) + } + + /// Returns a raw pointer to the inner C struct. + /// + /// It is up to the caller to use the raw pointer correctly. + pub fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for LockClassKey { + fn drop(self: Pin<&mut Self>) { + // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address + // hasn't changed. Thus, it's safe to pass it to unregister. + unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } + } +} + +/// Defines a new static lock class and returns a pointer to it. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{static_lock_class, Arc, SpinLock}; +/// +/// fn new_locked_int() -> Result>> { +/// Arc::pin_init(SpinLock::new( +/// 42, +/// c"new_locked_int", +/// static_lock_class!(), +/// ), GFP_KERNEL) +/// } +/// ``` +#[macro_export] +macro_rules! static_lock_class { + () => {{ + static CLASS: $crate::sync::LockClassKey = + // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop + // never runs on a static global. + unsafe { $crate::sync::LockClassKey::new_static() }; + $crate::prelude::Pin::static_ref(&CLASS) + }}; +} +pub use static_lock_class; + +/// Returns the given string, if one is provided, otherwise generates one based on the source code +/// location. +#[doc(hidden)] +#[macro_export] +macro_rules! optional_name { + () => { + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) + }; + ($name:literal) => { + $crate::c_str!($name) + }; +} diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs new file mode 100644 index 000000000..8ae0fe6f1 --- /dev/null +++ b/rust/kernel/sync/arc.rs @@ -0,0 +1,918 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A reference-counted pointer. +//! +//! This module implements a way for users to create reference-counted objects and pointers to +//! them. Such a pointer automatically increments and decrements the count, and drops the +//! underlying object when it reaches zero. It is also safe to use concurrently from multiple +//! threads. +//! +//! It is different from the standard library's [`Arc`] in a few ways: +//! 1. It is backed by the kernel's [`Refcount`] type. +//! 2. It does not support weak references, which allows it to be half the size. +//! 3. It saturates the reference count instead of aborting when it goes over a threshold. +//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. +//! 5. The object in [`Arc`] is pinned implicitly. +//! +//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html + +use crate::{ + alloc::{AllocError, Flags, KBox}, + ffi::c_void, + fmt, + init::InPlaceInit, + sync::Refcount, + try_init, + types::ForeignOwnable, +}; +use core::{ + alloc::Layout, + borrow::{Borrow, BorrowMut}, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, + ops::{Deref, DerefMut}, + pin::Pin, + ptr::NonNull, +}; +use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit}; + +mod std_vendor; + +/// A reference-counted pointer to an instance of `T`. +/// +/// The reference count is incremented when new instances of [`Arc`] are created, and decremented +/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. +/// +/// # Invariants +/// +/// The reference count on an instance of [`Arc`] is always non-zero. +/// The object pointed to by [`Arc`] is always pinned. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// // Create a refcounted instance of `Example`. +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// +/// // Get a new pointer to `obj` and increment the refcount. +/// let cloned = obj.clone(); +/// +/// // Assert that both `obj` and `cloned` point to the same underlying object. +/// assert!(core::ptr::eq(&*obj, &*cloned)); +/// +/// // Destroy `obj` and decrement its refcount. +/// drop(obj); +/// +/// // Check that the values are still accessible through `cloned`. +/// assert_eq!(cloned.a, 10); +/// assert_eq!(cloned.b, 20); +/// +/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Using `Arc` as the type of `self`: +/// +/// ``` +/// use kernel::sync::Arc; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// impl Example { +/// fn take_over(self: Arc) { +/// // ... +/// } +/// +/// fn use_reference(self: &Arc) { +/// // ... +/// } +/// } +/// +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// obj.use_reference(); +/// obj.take_over(); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Coercion from `Arc` to `Arc`: +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// trait MyTrait { +/// // Trait has a function whose `self` type is `Arc`. +/// fn example1(self: Arc) {} +/// +/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`. +/// fn example2(self: ArcBorrow<'_, Self>) {} +/// } +/// +/// struct Example; +/// impl MyTrait for Example {} +/// +/// // `obj` has type `Arc`. +/// let obj: Arc = Arc::new(Example, GFP_KERNEL)?; +/// +/// // `coerced` has type `Arc`. +/// let coerced: Arc = obj; +/// # Ok::<(), Error>(()) +/// ``` +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct Arc { + ptr: NonNull>, + // NB: this informs dropck that objects of type `ArcInner` may be used in ` as + // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in + // ` as Drop>::drop` and the distinction between `T` and `ArcInner` is not presently + // meaningful with respect to dropck - but this may change in the future so this is left here + // out of an abundance of caution. + // + // See + // for more detail on the semantics of dropck in the presence of `PhantomData`. + _p: PhantomData>, +} + +#[pin_data] +#[repr(C)] +struct ArcInner { + refcount: Refcount, + data: T, +} + +impl ArcInner { + /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must + /// not yet have been destroyed. + unsafe fn container_of(ptr: *const T) -> NonNull> { + let refcount_layout = Layout::new::(); + // SAFETY: The caller guarantees that the pointer is valid. + let val_layout = Layout::for_value(unsafe { &*ptr }); + // SAFETY: We're computing the layout of a real struct that existed when compiling this + // binary, so its layout is not so large that it can trigger arithmetic overflow. + let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; + + // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and + // `ArcInner` is the same since `ArcInner` is a struct with `T` as its last field. + // + // This is documented at: + // . + let ptr = ptr as *const ArcInner; + + // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the + // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is + // still valid. + let ptr = unsafe { ptr.byte_sub(val_offset) }; + + // SAFETY: The pointer can't be null since you can't have an `ArcInner` value at the null + // address. + unsafe { NonNull::new_unchecked(ptr.cast_mut()) } + } +} + +// SAFETY: It is safe to send `Arc` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has an `Arc` may ultimately access `T` using a +// mutable reference when the reference count reaches zero and `T` is dropped. +unsafe impl Send for Arc {} + +// SAFETY: It is safe to send `&Arc` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, +// it needs `T` to be `Send` because any thread that has a `&Arc` may clone it and get an +// `Arc` on that thread, so the thread may ultimately access `T` using a mutable reference when +// the reference count reaches zero and `T` is dropped. +unsafe impl Sync for Arc {} + +impl InPlaceInit for Arc { + type PinnedSelf = Self; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_pin_init(init, flags).map(|u| u.into()) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_init(init, flags).map(|u| u.into()) + } +} + +impl Arc { + /// Constructs a new reference counted instance of `T`. + pub fn new(contents: T, flags: Flags) -> Result { + // INVARIANT: The refcount is initialised to a non-zero value. + let value = ArcInner { + refcount: Refcount::new(1), + data: contents, + }; + + let inner = KBox::new(value, flags)?; + let inner = KBox::leak(inner).into(); + + // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new + // `Arc` object. + Ok(unsafe { Self::from_inner(inner) }) + } + + /// The offset that the value is stored at. + pub const DATA_OFFSET: usize = core::mem::offset_of!(ArcInner, data); +} + +impl Arc { + /// Constructs a new [`Arc`] from an existing [`ArcInner`]. + /// + /// # Safety + /// + /// The caller must ensure that `inner` points to a valid location and has a non-zero reference + /// count, one of which will be owned by the new [`Arc`] instance. + unsafe fn from_inner(inner: NonNull>) -> Self { + // INVARIANT: By the safety requirements, the invariants hold. + Arc { + ptr: inner, + _p: PhantomData, + } + } + + /// Convert the [`Arc`] into a raw pointer. + /// + /// The raw pointer has ownership of the refcount that this [`Arc`] object owned. + pub fn into_raw(self) -> *const T { + let ptr = self.ptr.as_ptr(); + core::mem::forget(self); + // SAFETY: The pointer is valid. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Return a raw pointer to the data in this [`Arc`]. + pub fn as_ptr(this: &Self) -> *const T { + let ptr = this.ptr.as_ptr(); + + // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`, + // field projection to `data`is within bounds of the allocation. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it + /// must not be called more than once for each previous call to [`Arc::into_raw`]. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the + // reference count held then will be owned by the new `Arc` object. + unsafe { Self::from_inner(ptr) } + } + + /// Returns an [`ArcBorrow`] from the given [`Arc`]. + /// + /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method + /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. + #[inline] + pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { + // SAFETY: The constraint that the lifetime of the shared reference must outlive that of + // the returned `ArcBorrow` ensures that the object remains alive and that no mutable + // reference can be created. + unsafe { ArcBorrow::new(self.ptr) } + } + + /// Compare whether two [`Arc`] pointers reference the same underlying object. + pub fn ptr_eq(this: &Self, other: &Self) -> bool { + core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) + } + + /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. + /// + /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that + /// this method will never call the destructor of the value. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let unique_arc = Arc::into_unique_or_drop(arc); + /// + /// // The above conversion should succeed since refcount of `arc` is 1. + /// assert!(unique_arc.is_some()); + /// + /// assert_eq!(*(unique_arc.unwrap()), 42); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let another = arc.clone(); + /// + /// let unique_arc = Arc::into_unique_or_drop(arc); + /// + /// // The above conversion should fail since refcount of `arc` is >1. + /// assert!(unique_arc.is_none()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn into_unique_or_drop(this: Self) -> Option>> { + // We will manually manage the refcount in this method, so we disable the destructor. + let this = ManuallyDrop::new(this); + // SAFETY: We own a refcount, so the pointer is still valid. + let refcount = unsafe { &this.ptr.as_ref().refcount }; + + // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will + // return without further touching the `Arc`. If the refcount reaches zero, then there are + // no other `Arc`s, and we can create a `UniqueArc`. + if refcount.dec_and_test() { + refcount.set(1); + + // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We + // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin + // their values. + Some(Pin::from(UniqueArc { + inner: ManuallyDrop::into_inner(this), + })) + } else { + None + } + } +} + +// SAFETY: The pointer returned by `into_foreign` was originally allocated as an +// `KBox>`, so that type is what determines the alignment. +unsafe impl ForeignOwnable for Arc { + const FOREIGN_ALIGN: usize = > as ForeignOwnable>::FOREIGN_ALIGN; + + type Borrowed<'a> = ArcBorrow<'a, T>; + type BorrowedMut<'a> = Self::Borrowed<'a>; + + fn into_foreign(self) -> *mut c_void { + ManuallyDrop::new(self).ptr.as_ptr().cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::>()) }; + + // SAFETY: By the safety requirement of this function, we know that `ptr` came from + // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and + // holds a reference count increment that is transferrable to us. + unsafe { Self::from_inner(inner) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::>()) }; + + // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive + // for the lifetime of the returned value. + unsafe { ArcBorrow::new(inner) } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { ::borrow(ptr) } + } +} + +impl Deref for Arc { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to dereference it. + unsafe { &self.ptr.as_ref().data } + } +} + +impl AsRef for Arc { + fn as_ref(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::Arc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Shared instance. +/// let arc = Arc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc.clone()); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for Arc { + fn borrow(&self) -> &T { + self.deref() + } +} + +impl Clone for Arc { + fn clone(&self) -> Self { + // INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero. + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to increment the refcount. + unsafe { self.ptr.as_ref() }.refcount.inc(); + + // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. + unsafe { Self::from_inner(self.ptr) } + } +} + +impl Drop for Arc { + fn drop(&mut self) { + // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and + // this instance is being dropped, so the broken invariant is not observable. + // SAFETY: By the type invariant, there is necessarily a reference to the object. + let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test(); + if is_zero { + // The count reached zero, we must free the memory. + // + // SAFETY: The pointer was initialised from the result of `KBox::leak`. + unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; + } + } +} + +impl From> for Arc { + fn from(item: UniqueArc) -> Self { + item.inner + } +} + +impl From>> for Arc { + fn from(item: Pin>) -> Self { + // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. + unsafe { Pin::into_inner_unchecked(item).inner } + } +} + +/// A borrowed reference to an [`Arc`] instance. +/// +/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler +/// to use just `&T`, which we can trivially get from an [`Arc`] instance. +/// +/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow` +/// over `&Arc` because the latter results in a double-indirection: a pointer (shared reference) +/// to a pointer ([`Arc`]) to the object (`T`). An [`ArcBorrow`] eliminates this double +/// indirection while still allowing one to increment the refcount and getting an [`Arc`] when/if +/// needed. +/// +/// # Invariants +/// +/// There are no mutable references to the underlying [`Arc`], and it remains valid for the +/// lifetime of the [`ArcBorrow`] instance. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// struct Example; +/// +/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc { +/// e.into() +/// } +/// +/// let obj = Arc::new(Example, GFP_KERNEL)?; +/// let cloned = do_something(obj.as_arc_borrow()); +/// +/// // Assert that both `obj` and `cloned` point to the same underlying object. +/// assert!(core::ptr::eq(&*obj, &*cloned)); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Using `ArcBorrow` as the type of `self`: +/// +/// ``` +/// use kernel::sync::{Arc, ArcBorrow}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// impl Example { +/// fn use_reference(self: ArcBorrow<'_, Self>) { +/// // ... +/// } +/// } +/// +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// obj.as_arc_borrow().use_reference(); +/// # Ok::<(), Error>(()) +/// ``` +#[repr(transparent)] +#[derive(core::marker::CoercePointee)] +pub struct ArcBorrow<'a, T: ?Sized + 'a> { + inner: NonNull>, + _p: PhantomData<&'a ()>, +} + +impl Clone for ArcBorrow<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for ArcBorrow<'_, T> {} + +impl ArcBorrow<'_, T> { + /// Creates a new [`ArcBorrow`] instance. + /// + /// # Safety + /// + /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance: + /// 1. That `inner` remains valid; + /// 2. That no mutable references to `inner` are created. + unsafe fn new(inner: NonNull>) -> Self { + // INVARIANT: The safety requirements guarantee the invariants. + Self { + inner, + _p: PhantomData, + } + } + + /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with + /// [`Arc::into_raw`] or [`Arc::as_ptr`]. + /// + /// # Safety + /// + /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`]. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must + /// not hit zero. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a + /// [`UniqueArc`] reference to this value. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: The caller promises that the value remains valid since the reference count must + // not hit zero, and no mutable reference will be created since that would involve a + // `UniqueArc`. + unsafe { Self::new(ptr) } + } +} + +impl From> for Arc { + fn from(b: ArcBorrow<'_, T>) -> Self { + // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop` + // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the + // increment. + ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) }) + .deref() + .clone() + } +} + +impl Deref for ArcBorrow<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant, the underlying object is still alive with no mutable + // references to it, so it is safe to create a shared reference. + unsafe { &self.inner.as_ref().data } + } +} + +/// A refcounted object that is known to have a refcount of 1. +/// +/// It is mutable and can be converted to an [`Arc`] so that it can be shared. +/// +/// # Invariants +/// +/// `inner` always has a reference count of 1. +/// +/// # Examples +/// +/// In the following example, we make changes to the inner object before turning it into an +/// `Arc` object (after which point, it cannot be mutated directly). Note that `x.into()` +/// cannot fail. +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; +/// x.a += 1; +/// x.b += 1; +/// Ok(x.into()) +/// } +/// +/// # test().unwrap(); +/// ``` +/// +/// In the following example we first allocate memory for a refcounted `Example` but we don't +/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], +/// followed by a conversion to `Arc`. This is particularly useful when allocation happens +/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let x = UniqueArc::new_uninit(GFP_KERNEL)?; +/// Ok(x.write(Example { a: 10, b: 20 }).into()) +/// } +/// +/// # test().unwrap(); +/// ``` +/// +/// In the last example below, the caller gets a pinned instance of `Example` while converting to +/// `Arc`; this is useful in scenarios where one needs a pinned reference during +/// initialisation, for example, when initialising fields that are wrapped in locks. +/// +/// ``` +/// use kernel::sync::{Arc, UniqueArc}; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn test() -> Result> { +/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); +/// // We can modify `pinned` because it is `Unpin`. +/// pinned.as_mut().a += 1; +/// Ok(pinned.into()) +/// } +/// +/// # test().unwrap(); +/// ``` +pub struct UniqueArc { + inner: Arc, +} + +impl InPlaceInit for UniqueArc { + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + UniqueArc::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + UniqueArc::new_uninit(flags)?.write_init(init) + } +} + +impl InPlaceWrite for UniqueArc> { + type Initialized = UniqueArc; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { pin_init::raw_try_init(slot, init)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} + +impl UniqueArc { + /// Tries to allocate a new [`UniqueArc`] instance. + pub fn new(value: T, flags: Flags) -> Result { + Ok(Self { + // INVARIANT: The newly-created object has a refcount of 1. + inner: Arc::new(value, flags)?, + }) + } + + /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. + pub fn new_uninit(flags: Flags) -> Result>, AllocError> { + // INVARIANT: The refcount is initialised to a non-zero value. + let inner = KBox::try_init::( + try_init!(ArcInner { + refcount: Refcount::new(1), + data <- pin_init::uninit::(), + }? AllocError), + flags, + )?; + Ok(UniqueArc { + // INVARIANT: The newly-created object has a refcount of 1. + // SAFETY: The pointer from the `KBox` is valid. + inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) }, + }) + } +} + +impl UniqueArc { + /// Return a raw pointer to the data in this [`UniqueArc`]. + #[inline] + pub fn as_ptr(this: &Self) -> *const T { + Arc::as_ptr(&this.inner) + } +} + +impl UniqueArc> { + /// Converts a `UniqueArc>` into a `UniqueArc` by writing a value into it. + pub fn write(mut self, value: T) -> UniqueArc { + self.deref_mut().write(value); + // SAFETY: We just wrote the value to be initialized. + unsafe { self.assume_init() } + } + + /// Unsafely assume that `self` is initialized. + /// + /// # Safety + /// + /// The caller guarantees that the value behind this pointer has been initialized. It is + /// *immediate* UB to call this when the value is not initialized. + pub unsafe fn assume_init(self) -> UniqueArc { + let inner = ManuallyDrop::new(self).inner.ptr; + UniqueArc { + // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be + // dropped). The types are compatible because `MaybeUninit` is compatible with `T`. + inner: unsafe { Arc::from_inner(inner.cast()) }, + } + } + + /// Initialize `self` using the given initializer. + #[inline] + pub fn init_with(mut self, init: impl Init) -> core::result::Result, E> { + // SAFETY: The supplied pointer is valid for initialization. + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }), + Err(err) => Err(err), + } + } + + /// Pin-initialize `self` using the given pin-initializer. + #[inline] + pub fn pin_init_with( + mut self, + init: impl PinInit, + ) -> core::result::Result>, E> { + // SAFETY: The supplied pointer is valid for initialization and we will later pin the value + // to ensure it does not move. + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }.into()), + Err(err) => Err(err), + } + } +} + +impl From> for Pin> { + fn from(obj: UniqueArc) -> Self { + // SAFETY: It is not possible to move/replace `T` inside a `Pin>` (unless `T` + // is `Unpin`), so it is ok to convert it to `Pin>`. + unsafe { Pin::new_unchecked(obj) } + } +} + +impl Deref for UniqueArc { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl DerefMut for UniqueArc { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so + // it is safe to dereference it. Additionally, we know there is only one reference when + // it's inside a `UniqueArc`, so it is safe to get a mutable reference. + unsafe { &mut self.inner.ptr.as_mut().data } + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::UniqueArc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl Borrow for UniqueArc { + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// # use kernel::sync::UniqueArc; +/// struct Foo>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl BorrowMut for UniqueArc { + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +impl fmt::Display for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Display for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Debug for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} + +impl fmt::Debug for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} diff --git a/rust/kernel/sync/arc/std_vendor.rs b/rust/kernel/sync/arc/std_vendor.rs new file mode 100644 index 000000000..11b3f4ecc --- /dev/null +++ b/rust/kernel/sync/arc/std_vendor.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . + +use crate::sync::{arc::ArcInner, Arc}; +use core::any::Any; + +impl Arc { + /// Attempt to downcast the `Arc` to a concrete type. + pub fn downcast(self) -> core::result::Result, Self> + where + T: Any + Send + Sync, + { + if (*self).is::() { + // SAFETY: We have just checked that the type is correct, so we can cast the pointer. + unsafe { + let ptr = self.ptr.cast::>(); + core::mem::forget(self); + Ok(Arc::from_inner(ptr)) + } + } else { + Err(self) + } + } +} diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs new file mode 100644 index 000000000..9983ee085 --- /dev/null +++ b/rust/kernel/sync/aref.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Internal reference counting support. +//! +//! Many C types already have their own reference counting mechanism (e.g. by storing a +//! `refcount_t`). This module provides support for directly using their internal reference count +//! from Rust; instead of making users have to use an additional Rust-reference count in the form of +//! [`Arc`]. +//! +//! The smart pointer [`ARef`] acts similarly to [`Arc`] in that it holds a refcount on the +//! underlying object, but this refcount is internal to the object. It essentially is a Rust +//! implementation of the `get_` and `put_` pattern used in C for reference counting. +//! +//! To make use of [`ARef`], `MyType` needs to implement [`AlwaysRefCounted`]. It is a trait +//! for accessing the internal reference count of an object of the `MyType` type. +//! +//! [`Arc`]: crate::sync::Arc +//! [`Arc`]: crate::sync::Arc + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + types::ForeignOwnable, // +}; + +/// Types that are _always_ reference counted. +/// +/// It allows such types to define their own custom ref increment and decrement functions. +/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference +/// [`ARef`]. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. For +/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted +/// instances of a type. +/// +/// # Safety +/// +/// Implementers must ensure that increments to the reference count keep the object alive in memory +/// at least until matching decrements are performed. +/// +/// Implementers must also ensure that all instances are reference-counted. (Otherwise they +/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object +/// alive.) +pub unsafe trait AlwaysRefCounted { + /// Increments the reference count on the object. + fn inc_ref(&self); + + /// Decrements the reference count on the object. + /// + /// Frees the object when the count reaches zero. + /// + /// # Safety + /// + /// Callers must ensure that there was a previous matching increment to the reference count, + /// and that the object is no longer used after its reference count is decremented (as it may + /// result in the object being freed), unless the caller owns another increment on the refcount + /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls + /// [`AlwaysRefCounted::dec_ref`] once). + unsafe fn dec_ref(obj: NonNull); +} + +/// An owned reference to an always-reference-counted object. +/// +/// The object's reference count is automatically decremented when an instance of [`ARef`] is +/// dropped. It is also automatically incremented when a new instance is created via +/// [`ARef::clone`]. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In +/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. +pub struct ARef { + ptr: NonNull, + _p: PhantomData, +} + +// SAFETY: It is safe to send `ARef` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has an `ARef` may ultimately access `T` using a +// mutable reference, for example, when the reference count reaches zero and `T` is dropped. +unsafe impl Send for ARef {} + +// SAFETY: It is safe to send `&ARef` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, +// it needs `T` to be `Send` because any thread that has a `&ARef` may clone it and get an +// `ARef` on that thread, so the thread may ultimately access `T` using a mutable reference, for +// example, when the reference count reaches zero and `T` is dropped. +unsafe impl Sync for ARef {} + +// Even if `T` is pinned, pointers to `T` can still move. +impl Unpin for ARef {} + +impl ARef { + /// Creates a new instance of [`ARef`]. + /// + /// It takes over an increment of the reference count on the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the reference count was incremented at least once, and that they + /// are properly relinquishing one increment. That is, if there is only one increment, callers + /// must not use the underlying object anymore -- it is only safe to do so via the newly + /// created [`ARef`]. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // increment on the refcount. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `ARef`, returning a raw pointer. + /// + /// This function does not change the refcount. After calling this function, the caller is + /// responsible for the refcount previously managed by the `ARef`. + /// + /// # Examples + /// + /// ``` + /// use core::ptr::NonNull; + /// use kernel::sync::aref::{ARef, AlwaysRefCounted}; + /// + /// struct Empty {} + /// + /// # // SAFETY: TODO. + /// unsafe impl AlwaysRefCounted for Empty { + /// fn inc_ref(&self) {} + /// unsafe fn dec_ref(_obj: NonNull) {} + /// } + /// + /// let mut data = Empty {}; + /// let ptr = NonNull::::new(&mut data).unwrap(); + /// # // SAFETY: TODO. + /// let data_ref: ARef = unsafe { ARef::from_raw(ptr) }; + /// let raw_ptr: NonNull = ARef::into_raw(data_ref); + /// + /// assert_eq!(ptr, raw_ptr); + /// ``` + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } +} + +impl Clone for ARef { + fn clone(&self) -> Self { + self.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(self.ptr) } + } +} + +impl Deref for ARef { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl From<&T> for ARef { + fn from(b: &T) -> Self { + b.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(NonNull::from(b)) } + } +} + +impl Drop for ARef { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to + // decrement. + unsafe { T::dec_ref(self.ptr) }; + } +} + +impl PartialEq> for ARef +where + T: AlwaysRefCounted + PartialEq, + U: AlwaysRefCounted, +{ + #[inline] + fn eq(&self, other: &ARef) -> bool { + T::eq(&**self, &**other) + } +} +impl Eq for ARef {} + +// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The +// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`. +unsafe impl ForeignOwnable for ARef { + const FOREIGN_ALIGN: usize = core::mem::align_of::(); + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a T + where + Self: 'a; + + #[inline] + fn into_foreign(self) -> *mut c_void { + ARef::into_raw(self).as_ptr().cast() + } + + #[inline] + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing + // the refcount, so we can transfer the ownership to the new `ARef`. + unsafe { ARef::from_raw(ptr) } + } + + #[inline] + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + #[inline] + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { ::borrow(ptr) } + } +} + +impl PartialEq<&'_ U> for ARef +where + T: AlwaysRefCounted + PartialEq, +{ + #[inline] + fn eq(&self, other: &&U) -> bool { + T::eq(&**self, other) + } +} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs new file mode 100644 index 000000000..9cd009d57 --- /dev/null +++ b/rust/kernel/sync/atomic.rs @@ -0,0 +1,850 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic primitives. +//! +//! These primitives have the same semantics as their C counterparts: and the precise definitions of +//! semantics can be found at [`LKMM`]. Note that Linux Kernel Memory (Consistency) Model is the +//! only model for Rust code in kernel, and Rust's own atomics should be avoided. +//! +//! # Data races +//! +//! [`LKMM`] atomics have different rules regarding data races: +//! +//! - A normal write from C side is treated as an atomic write if +//! CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=y. +//! - Mixed-size atomic accesses don't cause data races. +//! +//! [`LKMM`]: srctree/tools/memory-model/ + +mod internal; +pub mod ordering; +mod predefine; + +pub use internal::AtomicImpl; +pub use ordering::{Acquire, Full, Relaxed, Release}; + +pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps}; + +use crate::build_error; +use internal::AtomicRepr; +use ordering::OrderingType; + +/// A memory location which can be safely modified from multiple execution contexts. +/// +/// This has the same size, alignment and bit validity as the underlying type `T`. And it disables +/// niche optimization for the same reason as [`UnsafeCell`]. +/// +/// The atomic operations are implemented in a way that is fully compatible with the [Linux Kernel +/// Memory (Consistency) Model][LKMM], hence they should be modeled as the corresponding +/// [`LKMM`][LKMM] atomic primitives. With the help of [`Atomic::from_ptr()`] and +/// [`Atomic::as_ptr()`], this provides a way to interact with [C-side atomic operations] +/// (including those without the `atomic` prefix, e.g. `READ_ONCE()`, `WRITE_ONCE()`, +/// `smp_load_acquire()` and `smp_store_release()`). +/// +/// # Invariants +/// +/// `self.0` is a valid `T`. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +/// [LKMM]: srctree/tools/memory-model/ +/// [C-side atomic operations]: srctree/Documentation/atomic_t.txt +#[repr(transparent)] +pub struct Atomic(AtomicRepr); + +// SAFETY: `Atomic` is safe to transfer between execution contexts because of the safety +// requirement of `AtomicType`. +unsafe impl Send for Atomic {} + +// SAFETY: `Atomic` is safe to share among execution contexts because all accesses are atomic. +unsafe impl Sync for Atomic {} + +/// Types that support basic atomic operations. +/// +/// # Round-trip transmutability +/// +/// `T` is round-trip transmutable to `U` if and only if both of these properties hold: +/// +/// - Any valid bit pattern for `T` is also a valid bit pattern for `U`. +/// - Transmuting (e.g. using [`transmute()`]) a value of type `T` to `U` and then to `T` again +/// yields a value that is in all aspects equivalent to the original value. +/// +/// # Safety +/// +/// - [`Self`] must have the same size and alignment as [`Self::Repr`]. +/// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. +/// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is +/// automatically satisfied. The exception is pointer types that are even though marked as +/// `!Send` (e.g. raw pointers and [`NonNull`]) but requiring `unsafe` to do anything +/// meaningful on them. This is because transferring pointer values between execution contexts is +/// safe as long as the actual `unsafe` dereferencing is justified. +/// +/// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. +/// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic +/// variables over unit-only enums, see [Examples]. +/// +/// # Limitations +/// +/// Because C primitives are used to implement the atomic operations, and a C function requires a +/// valid object of a type to operate on (i.e. no `MaybeUninit<_>`), hence at the Rust <-> C +/// surface, only types with all the bits initialized can be passed. As a result, types like `(u8, +/// u16)` (padding bytes are uninitialized) are currently not supported. +/// +/// # Examples +/// +/// A unit-only enum that implements [`AtomicType`]: +/// +/// ``` +/// use kernel::sync::atomic::{AtomicType, Atomic, Relaxed}; +/// +/// #[derive(Clone, Copy, PartialEq, Eq)] +/// #[repr(i32)] +/// enum State { +/// Uninit = 0, +/// Working = 1, +/// Done = 2, +/// }; +/// +/// // SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip +/// // transmutable to `i32`. +/// unsafe impl AtomicType for State { +/// type Repr = i32; +/// } +/// +/// let s = Atomic::new(State::Uninit); +/// +/// assert_eq!(State::Uninit, s.load(Relaxed)); +/// ``` +/// [`transmute()`]: core::mem::transmute +/// [round-trip transmutable]: AtomicType#round-trip-transmutability +/// [Examples]: AtomicType#examples +/// [`NonNull`]: core::ptr::NonNull +pub unsafe trait AtomicType: Sized + Copy { + /// The backing atomic implementation type. + type Repr: AtomicImpl; +} + +/// Types that support atomic add operations. +/// +/// # Safety +/// +// TODO: Properly defines `wrapping_add` in the following comment. +/// `wrapping_add` any value of type `Self::Repr::Delta` obtained by [`Self::rhs_into_delta()`] to +/// any value of type `Self::Repr` obtained through transmuting a value of type `Self` to must +/// yield a value with a bit pattern also valid for `Self`. +pub unsafe trait AtomicAdd: AtomicType { + /// Converts `Rhs` into the `Delta` type of the atomic implementation. + fn rhs_into_delta(rhs: Rhs) -> ::Delta; +} + +#[inline(always)] +const fn into_repr(v: T) -> T::Repr { + // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to + // `T::Repr`, therefore the transmute operation is sound. + unsafe { core::mem::transmute_copy(&v) } +} + +/// # Safety +/// +/// `r` must be a valid bit pattern of `T`. +#[inline(always)] +const unsafe fn from_repr(r: T::Repr) -> T { + // SAFETY: Per the safety requirement of the function, the transmute operation is sound. + unsafe { core::mem::transmute_copy(&r) } +} + +impl Atomic { + /// Creates a new atomic `T`. + pub const fn new(v: T) -> Self { + // INVARIANT: Per the safety requirement of `AtomicType`, `into_repr(v)` is a valid `T`. + Self(AtomicRepr::new(into_repr(v))) + } + + /// Creates a reference to an atomic `T` from a pointer of `T`. + /// + /// This usually is used when communicating with C side or manipulating a C struct, see + /// examples below. + /// + /// # Safety + /// + /// - `ptr` is aligned to `align_of::()`. + /// - `ptr` is valid for reads and writes for `'a`. + /// - For the duration of `'a`, other accesses to `*ptr` must not cause data races (defined + /// by [`LKMM`]) against atomic operations on the returned reference. Note that if all other + /// accesses are atomic, then this safety requirement is trivially fulfilled. + /// + /// [`LKMM`]: srctree/tools/memory-model + /// + /// # Examples + /// + /// Using [`Atomic::from_ptr()`] combined with [`Atomic::load()`] or [`Atomic::store()`] can + /// achieve the same functionality as `READ_ONCE()`/`smp_load_acquire()` or + /// `WRITE_ONCE()`/`smp_store_release()` in C side: + /// + /// ``` + /// # use kernel::types::Opaque; + /// use kernel::sync::atomic::{Atomic, Relaxed, Release}; + /// + /// // Assume there is a C struct `foo`. + /// mod cbindings { + /// #[repr(C)] + /// pub(crate) struct foo { + /// pub(crate) a: i32, + /// pub(crate) b: i32 + /// } + /// } + /// + /// let tmp = Opaque::new(cbindings::foo { a: 1, b: 2 }); + /// + /// // struct foo *foo_ptr = ..; + /// let foo_ptr = tmp.get(); + /// + /// // SAFETY: `foo_ptr` is valid, and `.a` is in bounds. + /// let foo_a_ptr = unsafe { &raw mut (*foo_ptr).a }; + /// + /// // a = READ_ONCE(foo_ptr->a); + /// // + /// // SAFETY: `foo_a_ptr` is valid for read, and all other accesses on it is atomic, so no + /// // data race. + /// let a = unsafe { Atomic::from_ptr(foo_a_ptr) }.load(Relaxed); + /// # assert_eq!(a, 1); + /// + /// // smp_store_release(&foo_ptr->a, 2); + /// // + /// // SAFETY: `foo_a_ptr` is valid for writes, and all other accesses on it is atomic, so + /// // no data race. + /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); + /// ``` + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { + // CAST: `T` and `Atomic` have the same size, alignment and bit validity. + // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will + // live long enough. It's safe to return a `&Atomic` because function safety requirement + // guarantees other accesses won't cause data races. + unsafe { &*ptr.cast::() } + } + + /// Returns a pointer to the underlying atomic `T`. + /// + /// Note that use of the return pointer must not cause data races defined by [`LKMM`]. + /// + /// # Guarantees + /// + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::()`]). + /// + /// [`LKMM`]: srctree/tools/memory-model + /// [`align_of::()`]: core::mem::align_of + pub const fn as_ptr(&self) -> *mut T { + // GUARANTEE: Per the function guarantee of `AtomicRepr::as_ptr()`, the `self.0.as_ptr()` + // must be a valid and properly aligned pointer for `T::Repr`, and per the safety guarantee + // of `AtomicType`, it's a valid and properly aligned pointer of `T`. + self.0.as_ptr().cast() + } + + /// Returns a mutable reference to the underlying atomic `T`. + /// + /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let mut atomic_val = Atomic::new(0u32); + /// let val_mut = atomic_val.get_mut(); + /// *val_mut = 101; + /// assert_eq!(101, atomic_val.load(Relaxed)); + /// ``` + pub fn get_mut(&mut self) -> &mut T { + // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of + // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting + // result is a valid pointer of `T`. + // SAFETY: The pointer is valid per the CAST comment above, and the mutable reference + // guarantees exclusive access. + unsafe { &mut *self.0.as_ptr().cast() } + } +} + +impl Atomic +where + T::Repr: AtomicBasicOps, +{ + /// Loads the value from the atomic `T`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42i32); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// let x = Atomic::new(42i64); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_read", "atomic64_read"))] + #[inline(always)] + pub fn load(&self, _: Ordering) -> T { + let v = { + match Ordering::TYPE { + OrderingType::Relaxed => T::Repr::atomic_read(&self.0), + OrderingType::Acquire => T::Repr::atomic_read_acquire(&self.0), + _ => build_error!("Wrong ordering"), + } + }; + + // SAFETY: `v` comes from reading `self.0`, which is a valid `T` per the type invariants. + unsafe { from_repr(v) } + } + + /// Stores a value to the atomic `T`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42i32); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// x.store(43, Relaxed); + /// + /// assert_eq!(43, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_set", "atomic64_set"))] + #[inline(always)] + pub fn store(&self, v: T, _: Ordering) { + let v = into_repr(v); + + // INVARIANT: `v` is a valid `T`, and is stored to `self.0` by `atomic_set*()`. + match Ordering::TYPE { + OrderingType::Relaxed => T::Repr::atomic_set(&self.0, v), + OrderingType::Release => T::Repr::atomic_set_release(&self.0, v), + _ => build_error!("Wrong ordering"), + } + } +} + +impl core::fmt::Debug for Atomic +where + T::Repr: AtomicBasicOps, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(&self.load(Relaxed), f) + } +} + +impl Atomic +where + T::Repr: AtomicExchangeOps, +{ + /// Atomic exchange. + /// + /// Atomically updates `*self` to `v` and returns the old value of `*self`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// assert_eq!(42, x.xchg(52, Acquire)); + /// assert_eq!(52, x.load(Relaxed)); + /// ``` + #[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))] + #[inline(always)] + pub fn xchg(&self, v: T, _: Ordering) -> T { + let v = into_repr(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to + // `T`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_xchg(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } + + /// Atomic compare and exchange. + /// + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not + /// modified. + /// + /// Compare: The comparison is done via the byte level comparison between `*self` and `old`. + /// + /// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type + /// parameter indicates, and a failed one doesn't provide any ordering, the load part of a + /// failed cmpxchg is a [`Relaxed`] load. + /// + /// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`, + /// otherwise returns `Err(value)`, and `value` is the current value of `*self`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// // Checks whether cmpxchg succeeded. + /// let success = x.cmpxchg(52, 64, Relaxed).is_ok(); + /// # assert!(!success); + /// + /// // Checks whether cmpxchg failed. + /// let failure = x.cmpxchg(52, 64, Relaxed).is_err(); + /// # assert!(failure); + /// + /// // Uses the old value if failed, probably re-try cmpxchg. + /// match x.cmpxchg(52, 64, Relaxed) { + /// Ok(_) => { }, + /// Err(old) => { + /// // do something with `old`. + /// # assert_eq!(old, 42); + /// } + /// } + /// + /// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C. + /// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); + /// # assert_eq!(42, latest); + /// assert_eq!(64, x.load(Relaxed)); + /// ``` + /// + /// [`Relaxed`]: ordering::Relaxed + #[doc(alias( + "atomic_cmpxchg", + "atomic64_cmpxchg", + "atomic_try_cmpxchg", + "atomic64_try_cmpxchg", + "compare_exchange" + ))] + #[inline(always)] + pub fn cmpxchg( + &self, + mut old: T, + new: T, + o: Ordering, + ) -> Result { + // Note on code generation: + // + // try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined, + // the compiler is able to figure out that branch is not needed if the users don't care + // about whether the operation succeeds or not. One exception is on x86, due to commit + // 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the + // atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the + // success of cmpxchg and only wants to use the old value. For example, for code like: + // + // let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); + // + // It will still generate code: + // + // movl $0x40, %ecx + // movl $0x34, %eax + // lock + // cmpxchgl %ecx, 0x4(%rsp) + // jne 1f + // 2: + // ... + // 1: movl %eax, %ecx + // jmp 2b + // + // This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old" + // location in the C function is always safe to write. + if self.try_cmpxchg(&mut old, new, o) { + Ok(old) + } else { + Err(old) + } + } + + /// Atomic compare and exchange and returns whether the operation succeeds. + /// + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not + /// modified, `*old` is updated to the current value of `*self`. + /// + /// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`]. + /// + /// Returns `true` means the cmpxchg succeeds otherwise returns `false`. + #[inline(always)] + fn try_cmpxchg(&self, old: &mut T, new: T, _: Ordering) -> bool { + let mut tmp = into_repr(*old); + let new = into_repr(new); + + // INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is + // transmutable to `T`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new), + OrderingType::Acquire => { + T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new) + } + OrderingType::Release => { + T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new) + } + OrderingType::Relaxed => { + T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new) + } + } + }; + + // SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants. + *old = unsafe { from_repr(tmp) }; + + ret + } +} + +impl Atomic +where + T::Repr: AtomicArithmeticOps, +{ + /// Atomic add. + /// + /// Atomically updates `*self` to `(*self).wrapping_add(v)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let x = Atomic::new(42); + /// + /// assert_eq!(42, x.load(Relaxed)); + /// + /// x.add(12, Relaxed); + /// + /// assert_eq!(54, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn add(&self, v: Rhs, _: ordering::Relaxed) + where + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_add()` due to safety requirement of + // `AtomicAdd`. + T::Repr::atomic_add(&self.0, v); + } + + /// Atomic fetch and add. + /// + /// Atomically updates `*self` to `(*self).wrapping_add(v)`, and returns the value of `*self` + /// before the update. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_add(12, Acquire)); + /// assert_eq!(54, x.load(Relaxed)); + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_add(12, Full)); + /// assert_eq!(54, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn fetch_add(&self, v: Rhs, _: Ordering) -> T + where + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_add*()` due to safety requirement + // of `AtomicAdd`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_fetch_add(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_fetch_add_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_fetch_add_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_fetch_add_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } + + /// Atomic fetch and subtract. + /// + /// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self` + /// before the update. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Acquire)); + /// assert_eq!(30, x.load(Relaxed)); + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Full)); + /// assert_eq!(30, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn fetch_sub(&self, v: Rhs, _: Ordering) -> T + where + // Types that support addition also support subtraction. + T: AtomicAdd, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement + // of `AtomicAdd`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } +} + +#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] +#[repr(C)] +#[derive(Clone, Copy)] +struct Flag { + bool_field: bool, +} + +/// # Invariants +/// +/// `padding` must be all zeroes. +#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] +#[repr(C, align(4))] +#[derive(Clone, Copy)] +struct Flag { + #[cfg(target_endian = "big")] + padding: [u8; 3], + bool_field: bool, + #[cfg(target_endian = "little")] + padding: [u8; 3], +} + +impl Flag { + #[inline(always)] + const fn new(b: bool) -> Self { + // INVARIANT: `padding` is all zeroes. + Self { + bool_field: b, + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + padding: [0; 3], + } + } +} + +// SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip +// transmutable to the selected representation (`i8` or `i32`). +unsafe impl AtomicType for Flag { + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] + type Repr = i8; + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + type Repr = i32; +} + +/// An atomic flag type intended to be backed by performance-optimal integer type. +/// +/// The backing integer type is an implementation detail; it may vary by architecture and change +/// in the future. +/// +/// [`AtomicFlag`] is generally preferable to [`Atomic`] when you need read-modify-write +/// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic`] does +/// not save memory due to padding. On some architectures that do not support byte-sized atomic +/// RMW operations, RMW operations on [`Atomic`] are slower. +/// +/// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic`] is fine. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::atomic::{AtomicFlag, Relaxed}; +/// +/// let flag = AtomicFlag::new(false); +/// assert_eq!(false, flag.load(Relaxed)); +/// flag.store(true, Relaxed); +/// assert_eq!(true, flag.load(Relaxed)); +/// ``` +pub struct AtomicFlag(Atomic); + +impl AtomicFlag { + /// Creates a new atomic flag. + #[inline(always)] + pub const fn new(b: bool) -> Self { + Self(Atomic::new(Flag::new(b))) + } + + /// Returns a mutable reference to the underlying flag as a [`bool`]. + /// + /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; + /// + /// let mut atomic_flag = AtomicFlag::new(false); + /// assert_eq!(false, atomic_flag.load(Relaxed)); + /// *atomic_flag.get_mut() = true; + /// assert_eq!(true, atomic_flag.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut bool { + &mut self.0.get_mut().bool_field + } + + /// Loads the value from the atomic flag. + #[inline(always)] + pub fn load(&self, o: Ordering) -> bool { + self.0.load(o).bool_field + } + + /// Stores a value to the atomic flag. + #[inline(always)] + pub fn store(&self, v: bool, o: Ordering) { + self.0.store(Flag::new(v), o); + } + + /// Stores a value to the atomic flag and returns the previous value. + #[inline(always)] + pub fn xchg(&self, new: bool, o: Ordering) -> bool { + self.0.xchg(Flag::new(new), o).bool_field + } + + /// Store a value to the atomic flag if the current value is equal to `old`. + #[inline(always)] + pub fn cmpxchg( + &self, + old: bool, + new: bool, + o: Ordering, + ) -> Result { + match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { + Ok(_) => Ok(old), + Err(f) => Err(f.bool_field), + } + } +} + +/// Atomic load over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic. +#[doc(alias("READ_ONCE", "smp_load_acquire"))] +#[inline(always)] +pub unsafe fn atomic_load( + ptr: *mut T, + o: Ordering, +) -> T +where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent stores from kernel are atomic, hence no data race per + // LKMM. + unsafe { Atomic::from_ptr(ptr) }.load(o) +} + +/// Atomic store over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Release)` maps to `smp_store_release()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("WRITE_ONCE", "smp_store_release"))] +#[inline(always)] +pub unsafe fn atomic_store( + ptr: *mut T, + v: T, + o: Ordering, +) where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.store(v, o); +} + +/// Atomic exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[inline(always)] +pub unsafe fn xchg( + ptr: *mut T, + new: T, + o: Ordering, +) -> T +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.xchg(new, o) +} + +/// Atomic compare and exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("try_cmpxchg"))] +#[inline(always)] +pub unsafe fn cmpxchg( + ptr: *mut T, + old: T, + new: T, + o: Ordering, +) -> Result +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o) +} diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs new file mode 100644 index 000000000..9c8a7a203 --- /dev/null +++ b/rust/kernel/sync/atomic/internal.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic internal implementations. +//! +//! Provides 1:1 mapping to the C atomic operations. + +use crate::{ + bindings, + build_assert::static_assert, + macros::paste, // +}; +use core::cell::UnsafeCell; +use ffi::c_void; + +mod private { + /// Sealed trait marker to disable customized impls on atomic implementation traits. + pub trait Sealed {} +} + +// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`), +// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of +// lower-level C primitives. +impl private::Sealed for i8 {} +impl private::Sealed for i16 {} +impl private::Sealed for *const c_void {} +impl private::Sealed for i32 {} +impl private::Sealed for i64 {} + +/// A marker trait for types that implement atomic operations with C side primitives. +/// +/// This trait is sealed, and only types that map directly to the C side atomics +/// or can be implemented with lower-level C primitives are allowed to implement this: +/// +/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives. +/// - `i32` map to `atomic_t` +/// - `i64` map to `atomic64_t` +pub trait AtomicImpl: Sized + Copy + private::Sealed { + /// The type of the delta in arithmetic or logical operations. + /// + /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of + /// [`Self`], but it may be different for the atomic pointer type. + type Delta; +} + +// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()` +// hence the atomicity is only guaranteed against read-modify-write operations if the architecture +// supports native atomic RmW. +// +// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the +// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to +// be added. +static_assert!( + cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), + "The current implementation of atomic i8/i16/ptr relies on the architecure being \ + ARCH_SUPPORTS_ATOMIC_RMW" +); + +impl AtomicImpl for i8 { + type Delta = Self; +} + +impl AtomicImpl for i16 { + type Delta = Self; +} + +impl AtomicImpl for *const c_void { + type Delta = isize; +} + +// `atomic_t` implements atomic operations on `i32`. +impl AtomicImpl for i32 { + type Delta = Self; +} + +// `atomic64_t` implements atomic operations on `i64`. +impl AtomicImpl for i64 { + type Delta = Self; +} + +/// Atomic representation. +#[repr(transparent)] +pub struct AtomicRepr(UnsafeCell); + +impl AtomicRepr { + /// Creates a new atomic representation `T`. + pub const fn new(v: T) -> Self { + Self(UnsafeCell::new(v)) + } + + /// Returns a pointer to the underlying `T`. + /// + /// # Guarantees + /// + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::()`]). + pub const fn as_ptr(&self) -> *mut T { + // GUARANTEE: `self.0` is an `UnsafeCell`, therefore the pointer returned by `.get()` + // must be valid and properly aligned. + self.0.get() + } +} + +// This macro generates the function signature with given argument list and return type. +macro_rules! declare_atomic_method { + ( + $(#[doc=$doc:expr])* + $func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)? + ) => { + paste!( + $(#[doc = $doc])* + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?; + ); + }; + ( + $(#[doc=$doc:expr])* + $func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? + ) => { + paste!( + declare_atomic_method!( + $(#[doc = $doc])* + [< $func _ $variant >]($($arg_sig)*) $(-> $ret)? + ); + ); + + declare_atomic_method!( + $(#[doc = $doc])* + $func [$($rest)*]($($arg_sig)*) $(-> $ret)? + ); + }; + ( + $(#[doc=$doc:expr])* + $func:ident []($($arg_sig:tt)*) $(-> $ret:ty)? + ) => { + declare_atomic_method!( + $(#[doc = $doc])* + $func($($arg_sig)*) $(-> $ret)? + ); + } +} + +// This macro generates the function implementation with given argument list and return type, and it +// will replace "call(...)" expression with "$ctype _ $func" to call the real C function. +macro_rules! impl_atomic_method { + ( + ($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? { + $unsafe:tt { call($($c_arg:expr),*) } + } + ) => { + paste!( + #[inline(always)] + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? { + // TODO: Ideally we want to use the SAFETY comments written at the macro invocation + // (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments + // are just comments, and they are not passed to macros as tokens, therefore we + // cannot use them here. One potential improvement is that if we support using + // attributes as an alternative for SAFETY comments, then we can use that for macro + // generating code. + // + // SAFETY: specified on macro invocation. + $unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) } + } + ); + }; + ( + ($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? { + $unsafe:tt { call($($arg:tt)*) } + } + ) => { + paste!( + impl_atomic_method!( + ($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? { + $unsafe { call($($arg)*) } + } + ); + ); + impl_atomic_method!( + ($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? { + $unsafe { call($($arg)*) } + } + ); + }; + ( + ($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { call($($arg:tt)*) } + } + ) => { + impl_atomic_method!( + ($ctype) $func($($arg_sig)*) $(-> $ret)? { + $unsafe { call($($arg)*) } + } + ); + } +} + +macro_rules! declare_atomic_ops_trait { + ( + $(#[$attr:meta])* $pub:vis trait $ops:ident { + $( + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } + )* + } + ) => { + $(#[$attr])* + $pub trait $ops: AtomicImpl { + $( + declare_atomic_method!( + $(#[doc=$doc])* + $func[$($variant)*]($($arg_sig)*) $(-> $ret)? + ); + )* + } + } +} + +macro_rules! impl_atomic_ops_for_one { + ( + $ty:ty => $ctype:ident, + $(#[$attr:meta])* $pub:vis trait $ops:ident { + $( + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } + )* + } + ) => { + impl $ops for $ty { + $( + impl_atomic_method!( + ($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { + $unsafe { call($($arg)*) } + } + ); + )* + } + } +} + +// Declares $ops trait with methods and implements the trait. +macro_rules! declare_and_impl_atomic_methods { + ( + [ $($map:tt)* ] + $(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* } + ) => { + declare_and_impl_atomic_methods!( + @with_ops_def + [ $($map)* ] + ( $(#[$attr])* $pub trait $ops { $($body)* } ) + ); + }; + + (@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => { + declare_atomic_ops_trait!( $($ops_def)* ); + + declare_and_impl_atomic_methods!( + @munch + [ $($map)* ] + ( $($ops_def)* ) + ); + }; + + (@munch [] ( $($ops_def:tt)* )) => {}; + + (@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => { + impl_atomic_ops_for_one!( + $ty => $ctype, + $($ops_def)* + ); + + declare_and_impl_atomic_methods!( + @munch + [ $($($rest)*)? ] + ( $($ops_def)* ) + ); + }; +} + +declare_and_impl_atomic_methods!( + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] + /// Basic atomic operations + pub trait AtomicBasicOps { + /// Atomic read (load). + fn read[acquire](a: &AtomicRepr) -> Self { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast()) } + } + + /// Atomic set (store). + fn set[release](a: &AtomicRepr, v: Self) { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), v) } + } + } +); + +declare_and_impl_atomic_methods!( + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] + /// Exchange and compare-and-exchange atomic operations + pub trait AtomicExchangeOps { + /// Atomic exchange. + /// + /// Atomically updates `*a` to `v` and returns the old value. + fn xchg[acquire, release, relaxed](a: &AtomicRepr, v: Self) -> Self { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), v) } + } + + /// Atomic compare and exchange. + /// + /// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not + /// modified, `*old` is updated to the current value of `*a`. + /// + /// Return `true` if the update of `*a` occurred, `false` otherwise. + fn try_cmpxchg[acquire, release, relaxed]( + a: &AtomicRepr, old: &mut Self, new: Self + ) -> bool { + // SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)` + // is valid and properly aligned. + unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) } + } + } +); + +declare_and_impl_atomic_methods!( + [ i32 => atomic, i64 => atomic64 ] + /// Atomic arithmetic operations + pub trait AtomicArithmeticOps { + /// Atomic add (wrapping). + /// + /// Atomically updates `*a` to `(*a).wrapping_add(v)`. + fn add[](a: &AtomicRepr, v: Self::Delta) { + // SAFETY: `a.as_ptr()` is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + + /// Atomic fetch and add (wrapping). + /// + /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` + /// before the update. + fn fetch_add[acquire, release, relaxed](a: &AtomicRepr, v: Self::Delta) -> Self { + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + + fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr, v: Self::Delta) -> Self { + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + } +); diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs new file mode 100644 index 000000000..c4e732e72 --- /dev/null +++ b/rust/kernel/sync/atomic/ordering.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory orderings. +//! +//! The semantics of these orderings follows the [`LKMM`] definitions and rules. +//! +//! - [`Acquire`] provides ordering between the load part of the annotated operation and all the +//! following memory accesses, and if there is a store part, the store part has the [`Relaxed`] +//! ordering. +//! - [`Release`] provides ordering between all the preceding memory accesses and the store part of +//! the annotated operation, and if there is a load part, the load part has the [`Relaxed`] +//! ordering. +//! - [`Full`] means "fully-ordered", that is: +//! - It provides ordering between all the preceding memory accesses and the annotated operation. +//! - It provides ordering between the annotated operation and all the following memory accesses. +//! - It provides ordering between all the preceding memory accesses and all the following memory +//! accesses. +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). +//! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are +//! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. +//! +//! [`LKMM`]: srctree/tools/memory-model/ +//! [`explanation`]: srctree/tools/memory-model/Documentation/explanation.txt + +/// The annotation type for relaxed memory ordering, for the description of relaxed memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Relaxed; + +/// The annotation type for acquire memory ordering, for the description of acquire memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Acquire; + +/// The annotation type for release memory ordering, for the description of release memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Release; + +/// The annotation type for fully-ordered memory ordering, for the description fully-ordered memory +/// ordering, see [module-level documentation]. +/// +/// [module-level documentation]: crate::sync::atomic::ordering +pub struct Full; + +/// Describes the exact memory ordering. +#[doc(hidden)] +pub enum OrderingType { + /// Relaxed ordering. + Relaxed, + /// Acquire ordering. + Acquire, + /// Release ordering. + Release, + /// Fully-ordered. + Full, +} + +mod internal { + /// Sealed trait, can be only implemented inside atomic mod. + pub trait Sealed {} + + impl Sealed for super::Relaxed {} + impl Sealed for super::Acquire {} + impl Sealed for super::Release {} + impl Sealed for super::Full {} +} + +/// The trait bound for annotating operations that support any ordering. +pub trait Ordering: internal::Sealed { + /// Describes the exact memory ordering. + const TYPE: OrderingType; +} + +impl Ordering for Relaxed { + const TYPE: OrderingType = OrderingType::Relaxed; +} + +impl Ordering for Acquire { + const TYPE: OrderingType = OrderingType::Acquire; +} + +impl Ordering for Release { + const TYPE: OrderingType = OrderingType::Release; +} + +impl Ordering for Full { + const TYPE: OrderingType = OrderingType::Full; +} + +/// The trait bound for operations that only support acquire or relaxed ordering. +pub trait AcquireOrRelaxed: Ordering {} + +impl AcquireOrRelaxed for Acquire {} +impl AcquireOrRelaxed for Relaxed {} + +/// The trait bound for operations that only support release or relaxed ordering. +pub trait ReleaseOrRelaxed: Ordering {} + +impl ReleaseOrRelaxed for Release {} +impl ReleaseOrRelaxed for Relaxed {} diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs new file mode 100644 index 000000000..3d63f4079 --- /dev/null +++ b/rust/kernel/sync/atomic/predefine.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Pre-defined atomic types + +use crate::prelude::*; + +// Ensure size and alignment requirements are checked. +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); + +// SAFETY: `bool` has the same size and alignment as `i8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (false) and 1 (true). Those are valid `i8` values, so `bool` is +// round-trip transmutable to `i8`. +unsafe impl super::AtomicType for bool { + type Repr = i8; +} + +// SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i8 { + type Repr = i8; +} + +// SAFETY: `i16` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i16 { + type Repr = i16; +} + +// SAFETY: +// +// - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*mut T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl super::AtomicType for *mut T { + type Repr = *const c_void; +} + +// SAFETY: +// +// - `*const T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*const T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl super::AtomicType for *const T { + type Repr = *const c_void; +} + +// SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i32 { + type Repr = i32; +} + +// SAFETY: The wrapping add result of two `i32`s is a valid `i32`. +unsafe impl super::AtomicAdd for i32 { + fn rhs_into_delta(rhs: i32) -> i32 { + rhs + } +} + +// SAFETY: `i64` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i64 { + type Repr = i64; +} + +// SAFETY: The wrapping add result of two `i64`s is a valid `i64`. +unsafe impl super::AtomicAdd for i64 { + fn rhs_into_delta(rhs: i64) -> i64 { + rhs + } +} + +// Defines an internal type that always maps to the integer type which has the same size alignment +// as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to +// `isize_atomic_repr`, which also always implements `AtomicImpl`. +#[allow(non_camel_case_types)] +#[cfg(not(testlib))] +#[cfg(not(CONFIG_64BIT))] +type isize_atomic_repr = i32; +#[allow(non_camel_case_types)] +#[cfg(not(testlib))] +#[cfg(CONFIG_64BIT)] +type isize_atomic_repr = i64; + +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "32")] +type isize_atomic_repr = i32; +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "64")] +type isize_atomic_repr = i64; + +// Ensure size and alignment requirements are checked. +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); + +// SAFETY: `isize` has the same size and alignment with `isize_atomic_repr`, and is round-trip +// transmutable to `isize_atomic_repr`. +unsafe impl super::AtomicType for isize { + type Repr = isize_atomic_repr; +} + +// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. +unsafe impl super::AtomicAdd for isize { + fn rhs_into_delta(rhs: isize) -> isize_atomic_repr { + rhs as isize_atomic_repr + } +} + +// SAFETY: `u32` and `i32` has the same size and alignment, and `u32` is round-trip transmutable to +// `i32`. +unsafe impl super::AtomicType for u32 { + type Repr = i32; +} + +// SAFETY: The wrapping add result of two `i32`s is a valid `u32`. +unsafe impl super::AtomicAdd for u32 { + fn rhs_into_delta(rhs: u32) -> i32 { + rhs as i32 + } +} + +// SAFETY: `u64` and `i64` has the same size and alignment, and `u64` is round-trip transmutable to +// `i64`. +unsafe impl super::AtomicType for u64 { + type Repr = i64; +} + +// SAFETY: The wrapping add result of two `i64`s is a valid `u64`. +unsafe impl super::AtomicAdd for u64 { + fn rhs_into_delta(rhs: u64) -> i64 { + rhs as i64 + } +} + +// SAFETY: `usize` has the same size and alignment with `isize_atomic_repr`, and is round-trip +// transmutable to `isize_atomic_repr`. +unsafe impl super::AtomicType for usize { + type Repr = isize_atomic_repr; +} + +// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. +unsafe impl super::AtomicAdd for usize { + fn rhs_into_delta(rhs: usize) -> isize_atomic_repr { + rhs as isize_atomic_repr + } +} + +#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)] +#[macros::kunit_tests(rust_atomics)] +mod tests { + use super::super::*; + + // Call $fn($val) with each $type of $val. + macro_rules! for_each_type { + ($val:literal in [$($type:ty),*] $fn:expr) => { + $({ + let v: $type = $val; + + $fn(v); + })* + } + } + + #[test] + fn atomic_basic_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + assert_eq!(v, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); + }); + } + + #[test] + fn atomic_acquire_release_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + + x.store(v, Release); + assert_eq!(v, x.load(Acquire)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + unsafe { atomic_store(ptr, v, Release) }; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); + }); + } + + #[test] + fn atomic_xchg_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + let old = v; + let new = v + 1; + + assert_eq!(old, x.xchg(new, Full)); + assert_eq!(new, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(old, unsafe { xchg(ptr, new, Full) }); + assert_eq!(new, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_cmpxchg_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + let old = v; + let new = v + 1; + + assert_eq!(Err(old), x.cmpxchg(new, new, Full)); + assert_eq!(old, x.load(Relaxed)); + assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); + assert_eq!(new, x.load(Relaxed)); + }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); + assert_eq!(old, x.load(Relaxed)); + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); + assert_eq!(new, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_arithmetic_tests() { + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + + assert_eq!(v, x.fetch_add(12, Full)); + assert_eq!(v + 12, x.load(Relaxed)); + + x.add(13, Relaxed); + + assert_eq!(v + 25, x.load(Relaxed)); + }); + } + + #[test] + fn atomic_bool_tests() { + let x = Atomic::new(false); + + assert_eq!(false, x.load(Relaxed)); + x.store(true, Relaxed); + assert_eq!(true, x.load(Relaxed)); + + assert_eq!(true, x.xchg(false, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + + assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); + } + + #[test] + fn atomic_ptr_tests() { + let mut v = 42; + let mut u = 43; + let x = Atomic::new(&raw mut v); + + assert_eq!(x.load(Acquire), &raw mut v); + assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v)); + assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v)); + assert_eq!(x.load(Relaxed), &raw mut u); + + let x = Atomic::new(&raw const v); + + assert_eq!(x.load(Acquire), &raw const v); + assert_eq!( + x.cmpxchg(&raw const u, &raw const u, Relaxed), + Err(&raw const v) + ); + assert_eq!( + x.cmpxchg(&raw const v, &raw const u, Relaxed), + Ok(&raw const v) + ); + assert_eq!(x.load(Relaxed), &raw const u); + } + + #[test] + fn atomic_flag_tests() { + let mut flag = AtomicFlag::new(false); + + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(true, flag.load(Relaxed)); + + assert_eq!(true, flag.xchg(false, Relaxed)); + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); + assert_eq!(false, flag.load(Relaxed)); + } +} diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs new file mode 100644 index 000000000..1180695d5 --- /dev/null +++ b/rust/kernel/sync/barrier.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory barriers. +//! +//! These primitives have the same semantics as their C counterparts: and the precise definitions +//! of semantics can be found at [`LKMM`]. +//! +//! [`LKMM`]: srctree/tools/memory-model/ + +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + +/// A compiler barrier. +/// +/// A barrier that prevents compiler from reordering memory accesses across the barrier. +#[inline(always)] +pub(crate) fn barrier() { + // By default, Rust inline asms are treated as being able to access any memory or flags, hence + // it suffices as a compiler barrier. + // + // SAFETY: An empty asm block. + unsafe { core::arch::asm!("") }; +} + +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb(_: T) { + T::run() +} + +/// Memory barrier between CPUs. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb>(_: T) { + if cfg!(CONFIG_SMP) { + T::run() + } else { + barrier() + } +} + +/// Memory barrier between local CPU and bus-mastering devices. +/// +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() +} diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs new file mode 100644 index 000000000..35ff049ff --- /dev/null +++ b/rust/kernel/sync/completion.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Completion support. +//! +//! Reference: +//! +//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h) + +use crate::{bindings, prelude::*, types::Opaque}; + +/// Synchronization primitive to signal when a certain task has been completed. +/// +/// The [`Completion`] synchronization primitive signals when a certain task has been completed by +/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, Completion}; +/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +/// +/// #[pin_data] +/// struct MyTask { +/// #[pin] +/// work: Work, +/// #[pin] +/// done: Completion, +/// } +/// +/// impl_has_work! { +/// impl HasWork for MyTask { self.work } +/// } +/// +/// impl MyTask { +/// fn new() -> Result> { +/// let this = Arc::pin_init(pin_init!(MyTask { +/// work <- new_work!("MyTask::work"), +/// done <- Completion::new(), +/// }), GFP_KERNEL)?; +/// +/// let _ = workqueue::system().enqueue(this.clone()); +/// +/// Ok(this) +/// } +/// +/// fn wait_for_completion(&self) { +/// self.done.wait_for_completion(); +/// +/// pr_info!("Completion: task complete\n"); +/// } +/// } +/// +/// impl WorkItem for MyTask { +/// type Pointer = Arc; +/// +/// fn run(this: Arc) { +/// // process this task +/// this.done.complete_all(); +/// } +/// } +/// +/// let task = MyTask::new()?; +/// task.wait_for_completion(); +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data] +pub struct Completion { + #[pin] + inner: Opaque, +} + +// SAFETY: `Completion` is safe to be send to any task. +unsafe impl Send for Completion {} + +// SAFETY: `Completion` is safe to be accessed concurrently. +unsafe impl Sync for Completion {} + +impl Completion { + /// Create an initializer for a new [`Completion`]. + pub fn new() -> impl PinInit { + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::completion| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`. + unsafe { bindings::init_completion(slot) }; + }), + }) + } + + fn as_raw(&self) -> *mut bindings::completion { + self.inner.get() + } + + /// Signal all tasks waiting on this completion. + /// + /// This method wakes up all tasks waiting on this completion; after this operation the + /// completion is permanently done, i.e. signals all current and future waiters. + #[inline] + pub fn complete_all(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::complete_all(self.as_raw()) }; + } + + /// Wait for completion of a task. + /// + /// This method waits for the completion of a task; it is not interruptible and there is no + /// timeout. + /// + /// See also [`Completion::complete_all`]. + #[inline] + pub fn wait_for_completion(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::wait_for_completion(self.as_raw()) }; + } +} diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs new file mode 100644 index 000000000..69d58dfba --- /dev/null +++ b/rust/kernel/sync/condvar.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A condition variable. +//! +//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition +//! variable. + +use super::{lock::Backend, lock::Guard, LockClassKey}; +use crate::{ + ffi::{c_int, c_long}, + str::{CStr, CStrExt as _}, + task::{ + MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE, + }, + time::Jiffies, + types::Opaque, +}; +use core::{marker::PhantomPinned, pin::Pin, ptr}; +use pin_init::{pin_data, pin_init, PinInit}; + +/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_condvar { + ($($name:literal)?) => { + $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_condvar; + +/// A conditional variable. +/// +/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to +/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And +/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or +/// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up +/// spuriously. +/// +/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init!) and [`new_condvar`] macros. +/// +/// # Examples +/// +/// The following is an example of using a condvar with a mutex: +/// +/// ``` +/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex}; +/// +/// #[pin_data] +/// pub struct Example { +/// #[pin] +/// value: Mutex, +/// +/// #[pin] +/// value_changed: CondVar, +/// } +/// +/// /// Waits for `e.value` to become `v`. +/// fn wait_for_value(e: &Example, v: u32) { +/// let mut guard = e.value.lock(); +/// while *guard != v { +/// e.value_changed.wait(&mut guard); +/// } +/// } +/// +/// /// Increments `e.value` and notifies all potential waiters. +/// fn increment(e: &Example) { +/// *e.value.lock() += 1; +/// e.value_changed.notify_all(); +/// } +/// +/// /// Allocates a new boxed `Example`. +/// fn new_example() -> Result>> { +/// KBox::pin_init(pin_init!(Example { +/// value <- new_mutex!(0), +/// value_changed <- new_condvar!(), +/// }), GFP_KERNEL) +/// } +/// ``` +/// +/// [`struct wait_queue_head`]: srctree/include/linux/wait.h +#[pin_data] +pub struct CondVar { + #[pin] + pub(crate) wait_queue_head: Opaque, + + /// A condvar needs to be pinned because it contains a [`struct list_head`] that is + /// self-referential, so it cannot be safely moved once it is initialised. + /// + /// [`struct list_head`]: srctree/include/linux/types.h + #[pin] + _pin: PhantomPinned, +} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. +unsafe impl Send for CondVar {} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads +// concurrently. +unsafe impl Sync for CondVar {} + +impl CondVar { + /// Constructs a new condvar initialiser. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + pin_init!(Self { + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + wait_queue_head <- Opaque::ffi_init(|slot| unsafe { + bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } + + fn wait_internal( + &self, + wait_state: c_int, + guard: &mut Guard<'_, T, B>, + timeout_in_jiffies: c_long, + ) -> c_long { + let wait = Opaque::::uninit(); + + // SAFETY: `wait` points to valid memory. + unsafe { bindings::init_wait(wait.get()) }; + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { + bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state) + }; + + // SAFETY: Switches to another thread. The timeout can be any number. + let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) }); + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) }; + + ret + } + + /// Releases the lock and waits for a notification in uninterruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by + /// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up + /// spuriously. + pub fn wait(&self, guard: &mut Guard<'_, T, B>) { + self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may + /// wake up due to signals. It may also wake up spuriously. + /// + /// Returns whether there is a signal pending. + #[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible(&self, guard: &mut Guard<'_, T, B>) -> bool { + self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + crate::current!().signal_pending() + } + + /// Releases the lock and waits for a notification in interruptible and freezable mode. + /// + /// The process is allowed to be frozen during this sleep. No lock should be held when calling + /// this function, and there is a lockdep assertion for this. Freezing a task that holds a lock + /// can trivially deadlock vs another task that needs that lock to complete before it too can + /// hit freezable. + #[must_use = "wait_interruptible_freezable returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_freezable( + &self, + guard: &mut Guard<'_, T, B>, + ) -> bool { + self.wait_internal( + TASK_INTERRUPTIBLE | TASK_FREEZABLE, + guard, + MAX_SCHEDULE_TIMEOUT, + ); + crate::current!().signal_pending() + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal. + #[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_timeout( + &self, + guard: &mut Guard<'_, T, B>, + jiffies: Jiffies, + ) -> CondVarTimeoutResult { + let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT); + let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies); + + match (res as Jiffies, crate::current!().signal_pending()) { + (jiffies, true) => CondVarTimeoutResult::Signal { jiffies }, + (0, false) => CondVarTimeoutResult::Timeout, + (jiffies, false) => CondVarTimeoutResult::Woken { jiffies }, + } + } + + /// Calls the kernel function to notify the appropriate number of threads. + fn notify(&self, count: c_int) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { + bindings::__wake_up( + self.wait_queue_head.get(), + TASK_NORMAL, + count, + ptr::null_mut(), + ) + }; + } + + /// Calls the kernel function to notify one thread synchronously. + /// + /// This method behaves like `notify_one`, except that it hints to the scheduler that the + /// current thread is about to go to sleep, so it should schedule the target thread on the same + /// CPU. + #[inline] + pub fn notify_sync(&self) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; + } + + /// Wakes a single waiter up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + #[inline] + pub fn notify_one(&self) { + self.notify(1); + } + + /// Wakes all waiters up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + #[inline] + pub fn notify_all(&self) { + self.notify(0); + } +} + +/// The return type of `wait_timeout`. +pub enum CondVarTimeoutResult { + /// The timeout was reached. + Timeout, + /// Somebody woke us up. + Woken { + /// Remaining sleep duration. + jiffies: Jiffies, + }, + /// A signal occurred. + Signal { + /// Remaining sleep duration. + jiffies: Jiffies, + }, +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs new file mode 100644 index 000000000..10b6b5e9b --- /dev/null +++ b/rust/kernel/sync/lock.rs @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic kernel lock and guard. +//! +//! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes, +//! spinlocks, raw spinlocks) to be provided with minimal effort. + +use super::LockClassKey; +use crate::{ + str::{CStr, CStrExt as _}, + types::{NotThreadSafe, Opaque, ScopeGuard}, +}; +use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; +use pin_init::{pin_data, pin_init, PinInit, Wrapper}; + +pub mod mutex; +pub mod spinlock; + +pub(super) mod global; +pub use global::{GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; + +/// The "backend" of a lock. +/// +/// It is the actual implementation of the lock, without the need to repeat patterns used in all +/// locks. +/// +/// # Safety +/// +/// - Implementers must ensure that only one thread/CPU may access the protected data once the lock +/// is owned, that is, between calls to [`lock`] and [`unlock`]. +/// - Implementers must also ensure that [`relock`] uses the same locking method as the original +/// lock operation. +/// +/// [`lock`]: Backend::lock +/// [`unlock`]: Backend::unlock +/// [`relock`]: Backend::relock +pub unsafe trait Backend { + /// The state required by the lock. + type State; + + /// The state required to be kept between [`lock`] and [`unlock`]. + /// + /// [`lock`]: Backend::lock + /// [`unlock`]: Backend::unlock + type GuardState; + + /// Initialises the lock. + /// + /// # Safety + /// + /// `ptr` must be valid for write for the duration of the call, while `name` and `key` must + /// remain valid for read indefinitely. + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ); + + /// Acquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + #[must_use] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; + + /// Tries to acquire the lock. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn try_lock(ptr: *mut Self::State) -> Option; + + /// Releases the lock, giving up its ownership. + /// + /// # Safety + /// + /// It must only be called by the current owner of the lock. + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState); + + /// Reacquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or + /// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now. + unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) { + // SAFETY: The safety requirements ensure that the lock is initialised. + *guard_state = unsafe { Self::lock(ptr) }; + } + + /// Asserts that the lock is held using lockdep. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn assert_is_held(ptr: *mut Self::State); +} + +/// A mutual exclusion primitive. +/// +/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock +/// [`Backend`] specified as the generic parameter `B`. +#[repr(C)] +#[pin_data] +pub struct Lock { + /// The kernel lock object. + #[pin] + state: Opaque, + + /// Some locks are known to be self-referential (e.g., mutexes), while others are architecture + /// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case + /// some architecture uses self-references now or in the future. + #[pin] + _pin: PhantomPinned, + + /// The data protected by the lock. + #[pin] + pub(crate) data: UnsafeCell, +} + +// SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for Lock {} + +// SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl Sync for Lock {} + +impl Lock { + /// Constructs a new lock initialiser. + pub fn new( + t: impl PinInit, + name: &'static CStr, + key: Pin<&'static LockClassKey>, + ) -> impl PinInit { + pin_init!(Self { + data <- UnsafeCell::pin_init(t), + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } +} + +impl Lock<(), B> { + /// Constructs a [`Lock`] from a raw pointer. + /// + /// This can be useful for interacting with a lock which was initialised outside of Rust. + /// + /// # Safety + /// + /// The caller promises that `ptr` points to a valid initialised instance of [`State`] during + /// the whole lifetime of `'a`. + /// + /// [`State`]: Backend::State + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut B::State) -> &'a Self { + // SAFETY: + // - By the safety contract `ptr` must point to a valid initialised instance of `B::State` + // - Since the lock data type is `()` which is a ZST, `state` is the only non-ZST member of + // the struct + // - Combined with `#[repr(C)]`, this guarantees `Self` has an equivalent data layout to + // `B::State`. + unsafe { &*ptr.cast() } + } +} + +impl Lock { + /// Acquires the lock and gives the caller access to the data protected by it. + #[inline] + pub fn lock(&self) -> Guard<'_, T, B> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + let state = unsafe { B::lock(self.state.get()) }; + // SAFETY: The lock was just acquired. + unsafe { Guard::new(self, state) } + } + + /// Tries to acquire the lock. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + // `Option` is not `#[must_use]` even if `T` is, thus the attribute is needed here. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock(&self) -> Option> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + unsafe { B::try_lock(self.state.get()).map(|state| Guard::new(self, state)) } + } +} + +/// A lock guard. +/// +/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock +/// when a guard goes out of scope. It also provides a safe and convenient way to access the data +/// protected by the lock. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, T: ?Sized, B: Backend> { + pub(crate) lock: &'a Lock, + pub(crate) state: B::GuardState, + _not_send: NotThreadSafe, +} + +// SAFETY: `Guard` is sync when the data protected by the lock is also sync. +unsafe impl Sync for Guard<'_, T, B> {} + +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Returns the lock that this guard originates from. + /// + /// # Examples + /// + /// The following example shows how to use [`Guard::lock_ref()`] to assert the corresponding + /// lock is held. + /// + /// ``` + /// # use kernel::{new_spinlock, sync::lock::{Backend, Guard, Lock}}; + /// # use pin_init::stack_pin_init; + /// + /// fn assert_held(guard: &Guard<'_, T, B>, lock: &Lock) { + /// // Address-equal means the same lock. + /// assert!(core::ptr::eq(guard.lock_ref(), lock)); + /// } + /// + /// // Creates a new lock on the stack. + /// stack_pin_init!{ + /// let l = new_spinlock!(42) + /// } + /// + /// let g = l.lock(); + /// + /// // `g` originates from `l`. + /// assert_held(&g, &l); + /// ``` + pub fn lock_ref(&self) -> &'a Lock { + self.lock + } + + pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce() -> U) -> U { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + + let _relock = ScopeGuard::new(|| + // SAFETY: The lock was just unlocked above and is being relocked now. + unsafe { B::relock(self.lock.state.get(), &mut self.state) }); + + cb() + } + + /// Returns a pinned mutable reference to the protected data. + /// + /// The guard implements [`DerefMut`] when `T: Unpin`, so for [`Unpin`] + /// types [`DerefMut`] should be used instead of this function. + /// + /// [`DerefMut`]: core::ops::DerefMut + /// [`Unpin`]: core::marker::Unpin + /// + /// # Examples + /// + /// ``` + /// # use kernel::sync::{Mutex, MutexGuard}; + /// # use core::{pin::Pin, marker::PhantomPinned}; + /// struct Data(PhantomPinned); + /// + /// fn example(mutex: &Mutex) { + /// let mut data: MutexGuard<'_, Data> = mutex.lock(); + /// let mut data: Pin<&mut Data> = data.as_mut(); + /// } + /// ``` + pub fn as_mut(&mut self) -> Pin<&mut T> { + // SAFETY: `self.lock.data` is structurally pinned. + unsafe { Pin::new_unchecked(&mut *self.lock.data.get()) } + } +} + +impl core::ops::Deref for Guard<'_, T, B> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &*self.lock.data.get() } + } +} + +impl core::ops::DerefMut for Guard<'_, T, B> +where + T: Unpin, +{ + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &mut *self.lock.data.get() } + } +} + +impl Drop for Guard<'_, T, B> { + #[inline] + fn drop(&mut self) { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + } +} + +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Constructs a new immutable lock guard. + /// + /// # Safety + /// + /// The caller must ensure that it owns the lock. + #[inline] + pub unsafe fn new(lock: &'a Lock, state: B::GuardState) -> Self { + // SAFETY: The caller can only hold the lock if `Backend::init` has already been called. + unsafe { B::assert_is_held(lock.state.get()) }; + + Self { + lock, + state, + _not_send: NotThreadSafe, + } + } +} diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs new file mode 100644 index 000000000..ebb10521d --- /dev/null +++ b/rust/kernel/sync/lock/global.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Support for defining statics containing locks. + +use crate::{ + str::{CStr, CStrExt as _}, + sync::lock::{Backend, Guard, Lock}, + sync::{LockClassKey, LockedBy}, + types::Opaque, +}; +use core::{ + cell::UnsafeCell, + marker::{PhantomData, PhantomPinned}, + pin::Pin, +}; + +/// Trait implemented for marker types for global locks. +/// +/// See [`global_lock!`] for examples. +pub trait GlobalLockBackend { + /// The name for this global lock. + const NAME: &'static CStr; + /// Item type stored in this global lock. + type Item: 'static; + /// The backend used for this global lock. + type Backend: Backend + 'static; + /// The class for this global lock. + fn get_lock_class() -> Pin<&'static LockClassKey>; +} + +/// Type used for global locks. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLock { + inner: Lock, +} + +impl GlobalLock { + /// Creates a global lock. + /// + /// # Safety + /// + /// * Before any other method on this lock is called, [`Self::init`] must be called. + /// * The type `B` must not be used with any other lock. + pub const unsafe fn new(data: B::Item) -> Self { + Self { + inner: Lock { + state: Opaque::uninit(), + data: UnsafeCell::new(data), + _pin: PhantomPinned, + }, + } + } + + /// Initializes a global lock. + /// + /// # Safety + /// + /// Must not be called more than once on a given lock. + pub unsafe fn init(&'static self) { + // SAFETY: The pointer to `state` is valid for the duration of this call, and both `name` + // and `key` are valid indefinitely. The `state` is pinned since we have a `'static` + // reference to `self`. + // + // We have exclusive access to the `state` since the caller of `new` promised to call + // `init` before using any other methods. As `init` can only be called once, all other + // uses of this lock must happen after this call. + unsafe { + B::Backend::init( + self.inner.state.get(), + B::NAME.as_char_ptr(), + B::get_lock_class().as_ptr(), + ) + } + } + + /// Lock this global lock. + #[inline] + pub fn lock(&'static self) -> GlobalGuard { + GlobalGuard { + inner: self.inner.lock(), + } + } + + /// Try to lock this global lock. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock(&'static self) -> Option> { + Some(GlobalGuard { + inner: self.inner.try_lock()?, + }) + } +} + +/// A guard for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct GlobalGuard { + inner: Guard<'static, B::Item, B::Backend>, +} + +impl core::ops::Deref for GlobalGuard { + type Target = B::Item; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl core::ops::DerefMut for GlobalGuard +where + B::Item: Unpin, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +/// A version of [`LockedBy`] for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLockedBy { + _backend: PhantomData, + value: UnsafeCell, +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl Send for GlobalLockedBy +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy: Send, +{ +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl Sync for GlobalLockedBy +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy: Sync, +{ +} + +impl GlobalLockedBy { + /// Create a new [`GlobalLockedBy`]. + /// + /// The provided value will be protected by the global lock indicated by `B`. + pub fn new(val: T) -> Self { + Self { + value: UnsafeCell::new(val), + _backend: PhantomData, + } + } +} + +impl GlobalLockedBy { + /// Access the value immutably. + /// + /// The caller must prove shared access to the lock. + pub fn as_ref<'a>(&'a self, _guard: &'a GlobalGuard) -> &'a T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &*self.value.get() } + } + + /// Access the value mutably. + /// + /// The caller must prove shared exclusive to the lock. + pub fn as_mut<'a>(&'a self, _guard: &'a mut GlobalGuard) -> &'a mut T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &mut *self.value.get() } + } + + /// Access the value mutably directly. + /// + /// The caller has exclusive access to this `GlobalLockedBy`, so they do not need to hold the + /// lock. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } +} + +/// Defines a global lock. +/// +/// The global mutex must be initialized before first use. Usually this is done by calling +/// [`GlobalLock::init`] in the module initializer. +/// +/// # Examples +/// +/// A global counter: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_COUNTER: Mutex = 0; +/// } +/// +/// fn increment_counter() -> u32 { +/// let mut guard = MY_COUNTER.lock(); +/// *guard += 1; +/// *guard +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// // SAFETY: Called exactly once. +/// unsafe { MY_COUNTER.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +/// +/// A global mutex used to protect all instances of a given struct: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// use kernel::sync::{GlobalGuard, GlobalLockedBy}; +/// +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_MUTEX: Mutex<()> = (); +/// } +/// +/// /// All instances of this struct are protected by `MY_MUTEX`. +/// struct MyStruct { +/// my_counter: GlobalLockedBy, +/// } +/// +/// impl MyStruct { +/// /// Increment the counter in this instance. +/// /// +/// /// The caller must hold the `MY_MUTEX` mutex. +/// fn increment(&self, guard: &mut GlobalGuard) -> u32 { +/// let my_counter = self.my_counter.as_mut(guard); +/// *my_counter += 1; +/// *my_counter +/// } +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// // SAFETY: Called exactly once. +/// unsafe { MY_MUTEX.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +#[macro_export] +macro_rules! global_lock { + { + $(#[$meta:meta])* $pub:vis + unsafe(uninit) static $name:ident: $kind:ident<$valuety:ty> = $value:expr; + } => { + #[doc = ::core::concat!( + "Backend type used by [`", + ::core::stringify!($name), + "`](static@", + ::core::stringify!($name), + ")." + )] + #[allow(non_camel_case_types, unreachable_pub)] + $pub enum $name {} + + impl $crate::sync::lock::GlobalLockBackend for $name { + const NAME: &'static $crate::str::CStr = $crate::c_str!(::core::stringify!($name)); + type Item = $valuety; + type Backend = $crate::global_lock_inner!(backend $kind); + + fn get_lock_class() -> Pin<&'static $crate::sync::LockClassKey> { + $crate::static_lock_class!() + } + } + + $(#[$meta])* + $pub static $name: $crate::sync::lock::GlobalLock<$name> = { + // Defined here to be outside the unsafe scope. + let init: $valuety = $value; + + // SAFETY: + // * The user of this macro promises to initialize the macro before use. + // * We are only generating one static with this backend type. + unsafe { $crate::sync::lock::GlobalLock::new(init) } + }; + }; +} +pub use global_lock; + +#[doc(hidden)] +#[macro_export] +macro_rules! global_lock_inner { + (backend Mutex) => { + $crate::sync::lock::mutex::MutexBackend + }; + (backend SpinLock) => { + $crate::sync::lock::spinlock::SpinLockBackend + }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; +} diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs new file mode 100644 index 000000000..cda0203ef --- /dev/null +++ b/rust/kernel/sync/lock/mutex.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel mutex. +//! +//! This module allows Rust code to use the kernel's `struct mutex`. + +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_mutex; + +/// A mutual exclusion primitive. +/// +/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, +/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is +/// unlocked, at which point another thread will be allowed to wake up and make progress. +/// +/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. +/// +/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_mutex`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a mutex. +/// +/// ``` +/// use kernel::sync::{new_mutex, Mutex}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: Mutex, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_mutex!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a mutex despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::Mutex; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &Mutex) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`struct mutex`]: srctree/include/linux/mutex.h +pub type Mutex = super::Lock; + +/// A [`Guard`] acquired from locking a [`Mutex`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`Mutex`]. It will unlock +/// the [`Mutex`] upon being dropped. +/// +/// [`Guard`]: super::Guard +pub type MutexGuard<'a, T> = super::Guard<'a, T, MutexBackend>; + +/// A kernel `struct mutex` lock backend. +pub struct MutexBackend; + +// SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion. +unsafe impl super::Backend for MutexBackend { + type State = bindings::mutex; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__mutex_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::mutex_lock(ptr) }; + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the mutex. + unsafe { bindings::mutex_unlock(ptr) }; + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::mutex_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::mutex_assert_is_held(ptr) } + } +} diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs new file mode 100644 index 000000000..aafc80125 --- /dev/null +++ b/rust/kernel/sync/lock/spinlock.rs @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel spinlock. +//! +//! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; + +/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLock::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock; + +/// A spinlock. +/// +/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only +/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is +/// unlocked, at which point another CPU will be allowed to make progress. +/// +/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a spinlock. +/// +/// ``` +/// use kernel::sync::{new_spinlock, SpinLock}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: SpinLock, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_spinlock!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a spinlock despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::SpinLock; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &SpinLock) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`spinlock_t`]: srctree/include/linux/spinlock.h +pub type SpinLock = Lock; + +/// A kernel `spinlock_t` lock backend. +pub struct SpinLockBackend; + +/// A [`Guard`] acquired from locking a [`SpinLock`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock +/// the [`SpinLock`] upon being dropped. +pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. +/// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and access a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be disabled already. +/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount +/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrq = super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer = Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state = this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting for + state.value += 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata = Test::new()?; + + let _ = workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state = testdata.state.lock(); + state.waiter_ready = true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs new file mode 100644 index 000000000..fb4a1430b --- /dev/null +++ b/rust/kernel/sync/locked_by.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A wrapper for data protected by a lock that does not wrap it. + +use super::{lock::Backend, lock::Lock}; +use crate::build_assert::build_assert; +use core::{cell::UnsafeCell, mem::size_of, ptr}; + +/// Allows access to some data to be serialised by a lock that does not wrap it. +/// +/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., +/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible. +/// For example, if a container has a lock and some data in the contained elements needs +/// to be protected by the same lock. +/// +/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it +/// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence +/// refers to the wrong instance of the lock. +/// +/// [`Mutex`]: super::Mutex +/// [`SpinLock`]: super::SpinLock +/// +/// # Examples +/// +/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an +/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in +/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at +/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also +/// locked; we enforce at run time that the right `InnerDirectory` is locked. +/// +/// ``` +/// use kernel::sync::{LockedBy, Mutex}; +/// +/// struct InnerFile { +/// bytes_used: u64, +/// } +/// +/// struct File { +/// _ino: u32, +/// inner: LockedBy, +/// } +/// +/// struct InnerDirectory { +/// /// The sum of the bytes used by all files. +/// bytes_used: u64, +/// _files: KVec, +/// } +/// +/// struct Directory { +/// _ino: u32, +/// inner: Mutex, +/// } +/// +/// /// Prints `bytes_used` from both the directory and file. +/// fn print_bytes_used(dir: &Directory, file: &File) { +/// let guard = dir.inner.lock(); +/// let inner_file = file.inner.access(&guard); +/// pr_info!("{} {}\n", guard.bytes_used, inner_file.bytes_used); +/// } +/// +/// /// Increments `bytes_used` for both the directory and file. +/// fn inc_bytes_used(dir: &Directory, file: &File) { +/// let mut guard = dir.inner.lock(); +/// guard.bytes_used += 10; +/// +/// let file_inner = file.inner.access_mut(&mut guard); +/// file_inner.bytes_used += 10; +/// } +/// +/// /// Creates a new file. +/// fn new_file(ino: u32, dir: &Directory) -> File { +/// File { +/// _ino: ino, +/// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }), +/// } +/// } +/// ``` +pub struct LockedBy { + owner: *const U, + data: UnsafeCell, +} + +// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for LockedBy {} + +// SAFETY: If `T` is not `Sync`, then parallel shared access to this `LockedBy` allows you to use +// `access_mut` to hand out `&mut T` on one thread at the time. The requirement that `T: Send` is +// sufficient to allow that. +// +// If `T` is `Sync`, then the `access` method also becomes available, which allows you to obtain +// several `&T` from several threads at once. However, this is okay as `T` is `Sync`. +unsafe impl Sync for LockedBy {} + +impl LockedBy { + /// Constructs a new instance of [`LockedBy`]. + /// + /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure + /// that the right owner is being used to access the protected data. If the owner is freed, the + /// data becomes inaccessible; if another instance of the owner is allocated *on the same + /// memory location*, the data becomes accessible again: none of this affects memory safety + /// because in any case at most one thread (or CPU) can access the protected data at a time. + pub fn new(owner: &Lock, data: T) -> Self { + build_assert!( + size_of::>() > 0, + "The lock type cannot be a ZST because it may be impossible to distinguish instances" + ); + Self { + owner: owner.data.get(), + data: UnsafeCell::new(data), + } + } +} + +impl LockedBy { + /// Returns a reference to the protected data when the caller provides evidence (via a + /// reference) that the owner is locked. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches + /// the data protected by the lock without actually holding it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access<'a>(&'a self, owner: &'a U) -> &'a T + where + T: Sync, + { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there are only shared references to the owner for the + // duration of 'a, so it's not possible to use `Self::access_mut` to obtain a mutable + // reference to the inner value that aliases with this shared reference. The type is `Sync` + // so there are no other requirements. + unsafe { &*self.data.get() } + } + + /// Returns a mutable reference to the protected data when the caller provides evidence (via a + /// mutable owner) that the owner is locked mutably. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that + /// matches the data protected by the lock without actually holding it. + /// + /// Showing a mutable reference to the owner is sufficient because we know no other references + /// can exist to it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there is only one reference to the owner. + unsafe { &mut *self.data.get() } + } +} diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs new file mode 100644 index 000000000..dc40bfaa5 --- /dev/null +++ b/rust/kernel/sync/poll.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Utilities for working with `struct poll_table`. + +use crate::{ + alloc::AllocError, + bindings, + fs::File, + prelude::*, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // +}; + +/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_poll_condvar { + ($($name:literal)?) => { + $crate::sync::poll::PollCondVar::new( + $crate::optional_name!($($name)?), $crate::static_lock_class!() + ) + }; +} + +/// Wraps the kernel's `poll_table`. +/// +/// # Invariants +/// +/// The pointer must be null or reference a valid `poll_table`. +#[repr(transparent)] +pub struct PollTable<'a> { + table: *mut bindings::poll_table, + _lifetime: PhantomData<&'a bindings::poll_table>, +} + +impl<'a> PollTable<'a> { + /// Creates a [`PollTable`] from a valid pointer. + /// + /// # Safety + /// + /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`. + pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self { + // INVARIANTS: The safety requirements are the same as the struct invariants. + PollTable { + table, + _lifetime: PhantomData, + } + } + + /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified + /// using the condition variable. + pub fn register_wait(&self, file: &File, cv: &PollCondVar) { + // SAFETY: + // * `file.as_ptr()` references a valid file for the duration of this call. + // * `self.table` is null or references a valid poll_table for the duration of this call. + // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory + // containing `cv.wait_queue_head` is invalidated. Since the destructor clears all + // waiters and then waits for an rcu grace period, it's guaranteed that + // `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal + // of the last waiter. + unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) } + } +} + +/// A wrapper around [`CondVar`] that makes it usable with [`PollTable`]. +/// +/// [`CondVar`]: crate::sync::CondVar +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct PollCondVar { + #[pin] + inner: CondVar, +} + +impl PollCondVar { + /// Constructs a new condvar initialiser. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + pin_init!(Self { + inner <- CondVar::new(name, key), + }) + } +} + +// Make the `CondVar` methods callable on `PollCondVar`. +impl Deref for PollCondVar { + type Target = CondVar; + + fn deref(&self) -> &CondVar { + &self.inner + } +} + +#[pinned_drop] +impl PinnedDrop for PollCondVar { + #[inline] + fn drop(self: Pin<&mut Self>) { + // Clear anything registered using `register_wait`. + // + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; + + // Wait for epoll items to be properly removed. + synchronize_rcu(); + } +} + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result { + let b = KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target = PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual destructor of the type, + // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of + // `synchronize_rcu()`. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs new file mode 100644 index 000000000..0daa1ac87 --- /dev/null +++ b/rust/kernel/sync/rcu.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RCU support. +//! +//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h) + +use crate::{bindings, types::NotThreadSafe}; + +/// Evidence that the RCU read side lock is held on the current thread/CPU. +/// +/// The type is explicitly not `Send` because this property is per-thread/CPU. +/// +/// # Invariants +/// +/// The RCU read side lock is actually held while instances of this guard exist. +pub struct Guard(NotThreadSafe); + +impl Guard { + /// Acquires the RCU read side lock and returns a guard. + #[inline] + pub fn new() -> Self { + // SAFETY: An FFI call with no additional requirements. + unsafe { bindings::rcu_read_lock() }; + // INVARIANT: The RCU read side lock was just acquired above. + Self(NotThreadSafe) + } + + /// Explicitly releases the RCU read side lock. + #[inline] + pub fn unlock(self) {} +} + +impl Default for Guard { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl Drop for Guard { + #[inline] + fn drop(&mut self) { + // SAFETY: By the type invariants, the RCU read side is locked, so it is ok to unlock it. + unsafe { bindings::rcu_read_unlock() }; + } +} + +/// Acquires the RCU read side lock. +#[inline] +pub fn read_lock() -> Guard { + Guard::new() +} + +/// Wait until all in-flight `call_rcu()` callbacks complete. +/// +/// Note that this primitive does not necessarily wait for an RCU grace period +/// to complete. For example, if there are no RCU callbacks queued anywhere +/// in the system, then [`rcu_barrier()`] is within its rights to return +/// immediately, without waiting for anything, much less an RCU grace period. +/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods +/// beyond those that were already destined to be executed. +/// +/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all +/// pending lazy RCU callbacks. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn rcu_barrier() { + // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period. + unsafe { bindings::rcu_barrier() }; +} + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs new file mode 100644 index 000000000..23a5d201f --- /dev/null +++ b/rust/kernel/sync/refcount.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Atomic reference counting. +//! +//! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h) + +use crate::{ + build_assert::build_assert, + sync::atomic::Atomic, + types::Opaque, // +}; + +/// Atomic reference counter. +/// +/// This type is conceptually an atomic integer, but provides saturation semantics compared to +/// normal atomic integers. Values in the negative range when viewed as a signed integer are +/// saturation (bad) values. For details about the saturation semantics, please refer to top of +/// [`include/linux/refcount.h`](srctree/include/linux/refcount.h). +/// +/// Wraps the kernel's C `refcount_t`. +#[repr(transparent)] +pub struct Refcount(Opaque); + +impl Refcount { + /// Construct a new [`Refcount`] from an initial value. + /// + /// The initial value should be non-saturated. + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + pub fn new(value: i32) -> Self { + build_assert!(value >= 0, "initial value saturated"); + // SAFETY: There are no safety requirements for this FFI call. + Self(Opaque::new(unsafe { bindings::REFCOUNT_INIT(value) })) + } + + #[inline] + fn as_ptr(&self) -> *mut bindings::refcount_t { + self.0.get() + } + + /// Get the underlying atomic counter that backs the refcount. + /// + /// NOTE: Usage of this function is discouraged as it can circumvent the protections offered by + /// `refcount.h`. If there is no way to achieve the result using APIs in `refcount.h`, then + /// this function can be used. Otherwise consider adding a binding for the required API. + #[inline] + pub fn as_atomic(&self) -> &Atomic { + let ptr = self.0.get().cast(); + // SAFETY: `refcount_t` is a transparent wrapper of `atomic_t`, which is an atomic 32-bit + // integer that is layout-wise compatible with `Atomic`. All values are valid for + // `refcount_t`, despite some of the values being considered saturated and "bad". + unsafe { &*ptr } + } + + /// Set a refcount's value. + #[inline] + pub fn set(&self, value: i32) { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_set(self.as_ptr(), value) } + } + + /// Increment a refcount. + /// + /// It will saturate if overflows and `WARN`. It will also `WARN` if the refcount is 0, as this + /// represents a possible use-after-free condition. + /// + /// Provides no memory ordering, it is assumed that caller already has a reference on the + /// object. + #[inline] + pub fn inc(&self) { + // SAFETY: self is valid. + unsafe { bindings::refcount_inc(self.as_ptr()) } + } + + /// Decrement a refcount. + /// + /// It will `WARN` on underflow and fail to decrement when saturated. + /// + /// Provides release memory ordering, such that prior loads and stores are done + /// before. + #[inline] + pub fn dec(&self) { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_dec(self.as_ptr()) } + } + + /// Decrement a refcount and test if it is 0. + /// + /// It will `WARN` on underflow and fail to decrement when saturated. + /// + /// Provides release memory ordering, such that prior loads and stores are done + /// before, and provides an acquire ordering on success such that memory deallocation + /// must come after. + /// + /// Returns true if the resulting refcount is 0, false otherwise. + /// + /// # Notes + /// + /// A common pattern of using `Refcount` is to free memory when the reference count reaches + /// zero. This means that the reference to `Refcount` could become invalid after calling this + /// function. This is fine as long as the reference to `Refcount` is no longer used when this + /// function returns `false`. It is not necessary to use raw pointers in this scenario, see + /// . + #[inline] + #[must_use = "use `dec` instead if you do not need to test if it is 0"] + pub fn dec_and_test(&self) -> bool { + // SAFETY: `self.as_ptr()` is valid. + unsafe { bindings::refcount_dec_and_test(self.as_ptr()) } + } +} + +// SAFETY: `refcount_t` is thread-safe. +unsafe impl Send for Refcount {} + +// SAFETY: `refcount_t` is thread-safe. +unsafe impl Sync for Refcount {} diff --git a/rust/kernel/sync/set_once.rs b/rust/kernel/sync/set_once.rs new file mode 100644 index 000000000..139cef05e --- /dev/null +++ b/rust/kernel/sync/set_once.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A container that can be initialized at most once. + +use super::atomic::{ + ordering::{Acquire, Relaxed, Release}, + Atomic, +}; +use core::{cell::UnsafeCell, mem::MaybeUninit}; + +/// A container that can be populated at most once. Thread safe. +/// +/// Once the a [`SetOnce`] is populated, it remains populated by the same object for the +/// lifetime `Self`. +/// +/// # Invariants +/// +/// - `init` may only increase in value. +/// - `init` may only assume values in the range `0..=2`. +/// - `init == 0` if and only if `value` is uninitialized. +/// - `init == 1` if and only if there is exactly one thread with exclusive +/// access to `self.value`. +/// - `init == 2` if and only if `value` is initialized and valid for shared +/// access. +/// +/// # Example +/// +/// ``` +/// # use kernel::sync::SetOnce; +/// let value = SetOnce::new(); +/// assert_eq!(None, value.as_ref()); +/// +/// let status = value.populate(42u8); +/// assert_eq!(true, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// +/// let status = value.populate(101u8); +/// assert_eq!(false, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// ``` +pub struct SetOnce { + init: Atomic, + value: UnsafeCell>, +} + +impl Default for SetOnce { + fn default() -> Self { + Self::new() + } +} + +impl SetOnce { + /// Create a new [`SetOnce`]. + /// + /// The returned instance will be empty. + pub const fn new() -> Self { + // INVARIANT: The container is empty and we initialize `init` to `0`. + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + init: Atomic::new(0), + } + } + + /// Get a reference to the contained object. + /// + /// Returns [`None`] if this [`SetOnce`] is empty. + pub fn as_ref(&self) -> Option<&T> { + if self.init.load(Acquire) == 2 { + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // is initialized and valid for shared access. + Some(unsafe { &*self.value.get().cast() }) + } else { + None + } + } + + /// Populate the [`SetOnce`]. + /// + /// Returns `true` if the [`SetOnce`] was successfully populated. + pub fn populate(&self, value: T) -> bool { + // INVARIANT: If the swap succeeds: + // - We increase `init`. + // - We write the valid value `1` to `init`. + // - Only one thread can succeed in this write, so we have exclusive access after the + // write. + if let Ok(0) = self.init.cmpxchg(0, 1, Relaxed) { + // SAFETY: By the type invariants of `Self`, the fact that we succeeded in writing `1` + // to `self.init` means we obtained exclusive access to `self.value`. + unsafe { core::ptr::write(self.value.get().cast(), value) }; + // INVARIANT: + // - We increase `init`. + // - We write the valid value `2` to `init`. + // - We release our exclusive access to `self.value` and it is now valid for shared + // access. + self.init.store(2, Release); + true + } else { + false + } + } + + /// Get a copy of the contained object. + /// + /// Returns [`None`] if the [`SetOnce`] is empty. + pub fn copy(&self) -> Option + where + T: Copy, + { + self.as_ref().copied() + } +} + +impl Drop for SetOnce { + fn drop(&mut self) { + if *self.init.get_mut() == 2 { + let value = self.value.get_mut(); + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // contains a valid value. We have exclusive access, as we hold a `mut` reference to + // `self`. + unsafe { value.assume_init_drop() }; + } + } +} + +// SAFETY: `SetOnce` can be transferred across thread boundaries iff the data it contains can. +unsafe impl Send for SetOnce {} + +// SAFETY: `SetOnce` synchronises access to the inner value via atomic operations, +// so shared references are safe when `T: Sync`. Since the inner `T` may be dropped +// on any thread, we also require `T: Send`. +unsafe impl Sync for SetOnce {} diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000..723e5e277 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs new file mode 100644 index 000000000..3336df493 --- /dev/null +++ b/rust/kernel/task.rs @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Tasks (threads and processes). +//! +//! C header: [`include/linux/sched.h`](srctree/include/linux/sched.h). + +use crate::{ + bindings, + mm::MmWithUser, + pid_namespace::PidNamespace, + prelude::*, + sync::aref::ARef, + types::{NotThreadSafe, Opaque}, +}; +use core::{ + ops::Deref, + ptr, // +}; + +/// A sentinel value used for infinite timeouts. +pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX; + +/// Bitmask for tasks that are sleeping in an interruptible state. +pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in an uninterruptible state. +pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in a freezable state. +pub const TASK_FREEZABLE: c_int = bindings::TASK_FREEZABLE as c_int; +/// Convenience constant for waking up tasks regardless of whether they are in interruptible or +/// uninterruptible sleep. +pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint; + +/// Returns the currently running task. +#[macro_export] +macro_rules! current { + () => { + // SAFETY: This expression creates a temporary value that is dropped at the end of the + // caller's scope. The following mechanisms ensure that the resulting `&CurrentTask` cannot + // leave current task context: + // + // * To return to userspace, the caller must leave the current scope. + // * Operations such as `begin_new_exec()` are necessarily unsafe and the caller of + // `begin_new_exec()` is responsible for safety. + // * Rust abstractions for things such as a `kthread_use_mm()` scope must require the + // closure to be `Send`, so the `NotThreadSafe` field of `CurrentTask` ensures that the + // `&CurrentTask` cannot cross the scope in either direction. + unsafe { &*$crate::task::Task::current() } + }; +} + +/// Wraps the kernel's `struct task_struct`. +/// +/// # Invariants +/// +/// All instances are valid tasks created by the C portion of the kernel. +/// +/// Instances of this type are always refcounted, that is, a call to `get_task_struct` ensures +/// that the allocation remains valid at least until the matching call to `put_task_struct`. +/// +/// # Examples +/// +/// The following is an example of getting the PID of the current thread with zero additional cost +/// when compared to the C version: +/// +/// ``` +/// let pid = current!().pid(); +/// ``` +/// +/// Getting the PID of the current process, also zero additional cost: +/// +/// ``` +/// let pid = current!().group_leader().pid(); +/// ``` +/// +/// Getting the current task and storing it in some struct. The reference count is automatically +/// incremented when creating `State` and decremented when it is dropped: +/// +/// ``` +/// use kernel::{task::Task, sync::aref::ARef}; +/// +/// struct State { +/// creator: ARef, +/// index: u32, +/// } +/// +/// impl State { +/// fn new() -> Self { +/// Self { +/// creator: ARef::from(&**current!()), +/// index: 0, +/// } +/// } +/// } +/// ``` +#[repr(transparent)] +pub struct Task(pub(crate) Opaque); + +// SAFETY: By design, the only way to access a `Task` is via the `current` function or via an +// `ARef` obtained through the `AlwaysRefCounted` impl. This means that the only situation in +// which a `Task` can be accessed mutably is when the refcount drops to zero and the destructor +// runs. It is safe for that to happen on any thread, so it is ok for this type to be `Send`. +unsafe impl Send for Task {} + +// SAFETY: It's OK to access `Task` through shared references from other threads because we're +// either accessing properties that don't change (e.g., `pid`, `group_leader`) or that are properly +// synchronised by C code (e.g., `signal_pending`). +unsafe impl Sync for Task {} + +/// Represents the [`Task`] in the `current` global. +/// +/// This type exists to provide more efficient operations that are only valid on the current task. +/// For example, to retrieve the pid-namespace of a task, you must use rcu protection unless it is +/// the current task. +/// +/// # Invariants +/// +/// Each value of this type must only be accessed from the task context it was created within. +/// +/// Of course, every thread is in a different task context, but for the purposes of this invariant, +/// these operations also permanently leave the task context: +/// +/// * Returning to userspace from system call context. +/// * Calling `release_task()`. +/// * Calling `begin_new_exec()` in a binary format loader. +/// +/// Other operations temporarily create a new sub-context: +/// +/// * Calling `kthread_use_mm()` creates a new context, and `kthread_unuse_mm()` returns to the +/// old context. +/// +/// This means that a `CurrentTask` obtained before a `kthread_use_mm()` call may be used again +/// once `kthread_unuse_mm()` is called, but it must not be used between these two calls. +/// Conversely, a `CurrentTask` obtained between a `kthread_use_mm()`/`kthread_unuse_mm()` pair +/// must not be used after `kthread_unuse_mm()`. +#[repr(transparent)] +pub struct CurrentTask(Task, NotThreadSafe); + +// Make all `Task` methods available on `CurrentTask`. +impl Deref for CurrentTask { + type Target = Task; + #[inline] + fn deref(&self) -> &Task { + &self.0 + } +} + +/// The type of process identifiers (PIDs). +pub type Pid = bindings::pid_t; + +/// The type of user identifiers (UIDs). +#[derive(Copy, Clone)] +pub struct Kuid { + kuid: bindings::kuid_t, +} + +impl Task { + /// Returns a raw pointer to the current task. + /// + /// It is up to the user to use the pointer correctly. + #[inline] + pub fn current_raw() -> *mut bindings::task_struct { + // SAFETY: Getting the current pointer is always safe. + unsafe { bindings::get_current() } + } + + /// Returns a task reference for the currently executing task/thread. + /// + /// The recommended way to get the current task/thread is to use the + /// [`current`] macro because it is safe. + /// + /// # Safety + /// + /// Callers must ensure that the returned object is only used to access a [`CurrentTask`] + /// within the task context that was active when this function was called. For more details, + /// see the invariants section for [`CurrentTask`]. + #[inline] + pub unsafe fn current() -> impl Deref { + struct TaskRef { + task: *const CurrentTask, + } + + impl Deref for TaskRef { + type Target = CurrentTask; + + fn deref(&self) -> &Self::Target { + // SAFETY: The returned reference borrows from this `TaskRef`, so it cannot outlive + // the `TaskRef`, which the caller of `Task::current()` has promised will not + // outlive the task/thread for which `self.task` is the `current` pointer. Thus, it + // is okay to return a `CurrentTask` reference here. + unsafe { &*self.task } + } + } + + TaskRef { + // CAST: The layout of `struct task_struct` and `CurrentTask` is identical. + task: Task::current_raw().cast(), + } + } + + /// Returns a raw pointer to the task. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::task_struct { + self.0.get() + } + + /// Returns the PID of the given task. + pub fn pid(&self) -> Pid { + // SAFETY: The pid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).pid) } + } + + /// Returns the TGID (Thread Group ID / Process ID) of the given task. + pub fn tgid(&self) -> Pid { + // SAFETY: The tgid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).tgid) } + } + + /// Returns the objective real UID of the given task. + #[inline] + pub fn uid(&self) -> Kuid { + // SAFETY: It's always safe to call `task_uid` on a valid task. + Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) + } + + /// Determines whether the given task has pending signals. + #[inline] + pub fn signal_pending(&self) -> bool { + // SAFETY: It's always safe to call `signal_pending` on a valid task. + unsafe { bindings::signal_pending(self.as_ptr()) != 0 } + } + + /// Returns task's pid namespace with elevated reference count + #[inline] + pub fn get_pid_ns(&self) -> Option> { + // SAFETY: By the type invariant, we know that `self.0` is valid. + let ptr = unsafe { bindings::task_get_pid_ns(self.as_ptr()) }; + if ptr.is_null() { + None + } else { + // SAFETY: `ptr` is valid by the safety requirements of this function. And we own a + // reference count via `task_get_pid_ns()`. + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::pid_namespace`. + Some(unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(ptr.cast::())) }) + } + } + + /// Returns the given task's pid in the provided pid namespace. + #[doc(alias = "task_tgid_nr_ns")] + #[inline] + pub fn tgid_nr_ns(&self, pidns: Option<&PidNamespace>) -> Pid { + let pidns = match pidns { + Some(pidns) => pidns.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: By the type invariant, we know that `self.0` is valid. We received a valid + // PidNamespace that we can use as a pointer or we received an empty PidNamespace and + // thus pass a null pointer. The underlying C function is safe to be used with NULL + // pointers. + unsafe { bindings::task_tgid_nr_ns(self.as_ptr(), pidns) } + } + + /// Wakes up the task. + #[inline] + pub fn wake_up(&self) { + // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task + // running. + unsafe { bindings::wake_up_process(self.as_ptr()) }; + } +} + +impl CurrentTask { + /// Access the address space of the current task. + /// + /// This function does not touch the refcount of the mm. + #[inline] + pub fn mm(&self) -> Option<&MmWithUser> { + // SAFETY: The `mm` field of `current` is not modified from other threads, so reading it is + // not a data race. + let mm = unsafe { (*self.as_ptr()).mm }; + + if mm.is_null() { + return None; + } + + // SAFETY: If `current->mm` is non-null, then it references a valid mm with a non-zero + // value of `mm_users`. Furthermore, the returned `&MmWithUser` borrows from this + // `CurrentTask`, so it cannot escape the scope in which the current pointer was obtained. + // + // This is safe even if `kthread_use_mm()`/`kthread_unuse_mm()` are used. There are two + // relevant cases: + // * If the `&CurrentTask` was created before `kthread_use_mm()`, then it cannot be + // accessed during the `kthread_use_mm()`/`kthread_unuse_mm()` scope due to the + // `NotThreadSafe` field of `CurrentTask`. + // * If the `&CurrentTask` was created within a `kthread_use_mm()`/`kthread_unuse_mm()` + // scope, then the `&CurrentTask` cannot escape that scope, so the returned `&MmWithUser` + // also cannot escape that scope. + // In either case, it's not possible to read `current->mm` and keep using it after the + // scope is ended with `kthread_unuse_mm()`. + Some(unsafe { MmWithUser::from_raw(mm) }) + } + + /// Access the pid namespace of the current task. + /// + /// This function does not touch the refcount of the namespace or use RCU protection. + /// + /// To access the pid namespace of another task, see [`Task::get_pid_ns`]. + #[doc(alias = "task_active_pid_ns")] + #[inline] + pub fn active_pid_ns(&self) -> Option<&PidNamespace> { + // SAFETY: It is safe to call `task_active_pid_ns` without RCU protection when calling it + // on the current task. + let active_ns = unsafe { bindings::task_active_pid_ns(self.as_ptr()) }; + + if active_ns.is_null() { + return None; + } + + // The lifetime of `PidNamespace` is bound to `Task` and `struct pid`. + // + // The `PidNamespace` of a `Task` doesn't ever change once the `Task` is alive. + // + // From system call context retrieving the `PidNamespace` for the current task is always + // safe and requires neither RCU locking nor a reference count to be held. Retrieving the + // `PidNamespace` after `release_task()` for current will return `NULL` but no codepath + // like that is exposed to Rust. + // + // SAFETY: If `current`'s pid ns is non-null, then it references a valid pid ns. + // Furthermore, the returned `&PidNamespace` borrows from this `CurrentTask`, so it cannot + // escape the scope in which the current pointer was obtained, e.g. it cannot live past a + // `release_task()` call. + Some(unsafe { PidNamespace::from_ptr(active_ns) }) + } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: The type invariants guarantee that `Task` is always refcounted. +unsafe impl crate::sync::aref::AlwaysRefCounted for Task { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_task_struct(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } + } +} + +impl PartialEq for Task { + #[inline] + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.as_ptr(), other.as_ptr()) + } +} + +impl Eq for Task {} + +impl Kuid { + /// Get the current subjective effective UID. + #[inline] + pub fn current_euid() -> Kuid { + // SAFETY: Just an FFI call. + Self::from_raw(unsafe { bindings::current_euid() }) + } + + /// Create a `Kuid` given the raw C type. + #[inline] + pub fn from_raw(kuid: bindings::kuid_t) -> Self { + Self { kuid } + } + + /// Turn this kuid into the raw C type. + #[inline] + pub fn into_raw(self) -> bindings::kuid_t { + self.kuid + } + + /// Converts this kernel UID into a userspace UID. + /// + /// Uses the namespace of the current task. + #[inline] + pub fn into_uid_in_current_ns(self) -> bindings::uid_t { + // SAFETY: Just an FFI call. + unsafe { bindings::from_kuid(bindings::current_user_ns(), self.kuid) } + } +} + +impl PartialEq for Kuid { + #[inline] + fn eq(&self, other: &Kuid) -> bool { + // SAFETY: Just an FFI call. + unsafe { bindings::uid_eq(self.kuid, other.kuid) } + } +} + +impl Eq for Kuid {} + +/// Annotation for functions that can sleep. +/// +/// Equivalent to the C side [`might_sleep()`], this function serves as +/// a debugging aid and a potential scheduling point. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`might_sleep()`]: https://docs.kernel.org/driver-api/basics.html#c.might_sleep +#[track_caller] +#[inline] +pub fn might_sleep() { + #[cfg(CONFIG_DEBUG_ATOMIC_SLEEP)] + { + let loc = core::panic::Location::caller(); + let file = kernel::file_from_location(loc); + + // SAFETY: `file.as_ptr()` is valid for reading and guaranteed to be nul-terminated. + unsafe { crate::bindings::__might_sleep(file.as_char_ptr(), loc.line() as i32) } + } + + // SAFETY: Always safe to call. + unsafe { crate::bindings::might_resched() } +} diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs new file mode 100644 index 000000000..6c0a5e809 --- /dev/null +++ b/rust/kernel/time.rs @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Time related primitives. +//! +//! This module contains the kernel APIs related to time and timers that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! There are two types in this module: +//! +//! - The [`Instant`] type represents a specific point in time. +//! - The [`Delta`] type represents a span of time. +//! +//! Note that the C side uses `ktime_t` type to represent both. However, timestamp +//! and timedelta are different. To avoid confusion, we use two different types. +//! +//! A [`Instant`] object can be created by calling the [`Instant::now()`] function. +//! It represents a point in time at which the object was created. +//! By calling the [`Instant::elapsed()`] method, a [`Delta`] object representing +//! the elapsed time can be created. The [`Delta`] object can also be created +//! by subtracting two [`Instant`] objects. +//! +//! A [`Delta`] type supports methods to retrieve the duration in various units. +//! +//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). +//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). + +use core::marker::PhantomData; +use core::ops; + +pub mod delay; +pub mod hrtimer; + +/// The number of nanoseconds per microsecond. +pub const NSEC_PER_USEC: i64 = bindings::NSEC_PER_USEC as i64; + +/// The number of nanoseconds per millisecond. +pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; + +/// The number of nanoseconds per second. +pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64; + +/// The time unit of Linux kernel. One jiffy equals (1/HZ) second. +pub type Jiffies = crate::ffi::c_ulong; + +/// The millisecond time unit. +pub type Msecs = crate::ffi::c_uint; + +/// Converts milliseconds to jiffies. +#[inline] +pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { + // SAFETY: The `__msecs_to_jiffies` function is always safe to call no + // matter what the argument is. + unsafe { bindings::__msecs_to_jiffies(msecs) } +} + +/// Trait for clock sources. +/// +/// Selection of the clock source depends on the use case. In some cases the usage of a +/// particular clock is mandatory, e.g. in network protocols, filesystems. In other +/// cases the user of the clock has to decide which clock is best suited for the +/// purpose. In most scenarios clock [`Monotonic`] is the best choice as it +/// provides a accurate monotonic notion of time (leap second smearing ignored). +/// +/// # Safety +/// +/// Implementers must ensure that `ktime_get()` returns a value in the inclusive range +/// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to +/// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`). +pub unsafe trait ClockSource { + /// The kernel clock ID associated with this clock source. + /// + /// This constant corresponds to the C side `clockid_t` value. + const ID: bindings::clockid_t; + + /// Get the current time from the clock source. + /// + /// The function must return a value in the range `0..=KTIME_MAX`. + fn ktime_get() -> bindings::ktime_t; +} + +/// A monotonically increasing clock. +/// +/// A nonsettable system-wide clock that represents monotonic time since as +/// described by POSIX, "some unspecified point in the past". On Linux, that +/// point corresponds to the number of seconds that the system has been +/// running since it was booted. +/// +/// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the +/// CLOCK_REAL (e.g., if the system administrator manually changes the +/// clock), but is affected by frequency adjustments. This clock does not +/// count time that the system is suspended. +pub struct Monotonic; + +// SAFETY: The kernel's `ktime_get()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Monotonic { + const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get()` outside of NMI context. + unsafe { bindings::ktime_get() } + } +} + +/// A settable system-wide clock that measures real (i.e., wall-clock) time. +/// +/// Setting this clock requires appropriate privileges. This clock is +/// affected by discontinuous jumps in the system time (e.g., if the system +/// administrator manually changes the clock), and by frequency adjustments +/// performed by NTP and similar applications via adjtime(3), adjtimex(2), +/// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the +/// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time +/// (UTC) except that it ignores leap seconds; near a leap second it may be +/// adjusted by leap second smearing to stay roughly in sync with UTC. Leap +/// second smearing applies frequency adjustments to the clock to speed up +/// or slow down the clock to account for the leap second without +/// discontinuities in the clock. If leap second smearing is not applied, +/// the clock will experience discontinuity around leap second adjustment. +pub struct RealTime; + +// SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for RealTime { + const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_real()` outside of NMI context. + unsafe { bindings::ktime_get_real() } + } +} + +/// A monotonic that ticks while system is suspended. +/// +/// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC, +/// except that it also includes any time that the system is suspended. This +/// allows applications to get a suspend-aware monotonic clock without +/// having to deal with the complications of CLOCK_REALTIME, which may have +/// discontinuities if the time is changed using settimeofday(2) or similar. +pub struct BootTime; + +// SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for BootTime { + const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_boottime()` outside of NMI context. + unsafe { bindings::ktime_get_boottime() } + } +} + +/// International Atomic Time. +/// +/// A system-wide clock derived from wall-clock time but counting leap seconds. +/// +/// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is +/// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This +/// usually happens during boot and **should** not happen during normal operations. +/// However, if NTP or another application adjusts CLOCK_REALTIME by leap second +/// smearing, this clock will not be precise during leap second smearing. +/// +/// The acronym TAI refers to International Atomic Time. +pub struct Tai; + +// SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Tai { + const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_tai()` outside of NMI context. + unsafe { bindings::ktime_get_clocktai() } + } +} + +/// A specific point in time. +/// +/// # Invariants +/// +/// The `inner` value is in the range from 0 to `KTIME_MAX`. +#[repr(transparent)] +#[derive(PartialEq, PartialOrd, Eq, Ord)] +pub struct Instant { + inner: bindings::ktime_t, + _c: PhantomData, +} + +impl Clone for Instant { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Instant {} + +impl Instant { + /// Get the current time from the clock source. + #[inline] + pub fn now() -> Self { + // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range + // from 0 to `KTIME_MAX`. + Self { + inner: C::ktime_get(), + _c: PhantomData, + } + } + + /// Return the amount of time elapsed since the [`Instant`]. + #[inline] + pub fn elapsed(&self) -> Delta { + Self::now() - *self + } + + #[inline] + pub(crate) fn as_nanos(&self) -> i64 { + self.inner + } + + /// Create an [`Instant`] from a `ktime_t` without checking if it is non-negative. + /// + /// # Panics + /// + /// On debug builds, this function will panic if `ktime` is not in the range from 0 to + /// `KTIME_MAX`. + /// + /// # Safety + /// + /// The caller promises that `ktime` is in the range from 0 to `KTIME_MAX`. + #[inline] + pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self { + debug_assert!(ktime >= 0); + + // INVARIANT: Our safety contract ensures that `ktime` is in the range from 0 to + // `KTIME_MAX`. + Self { + inner: ktime, + _c: PhantomData, + } + } +} + +impl ops::Sub for Instant { + type Output = Delta; + + // By the type invariant, it never overflows. + #[inline] + fn sub(self, other: Instant) -> Delta { + Delta { + value: self.inner - other.inner, + } + } +} + +impl ops::Add for Instant { + type Output = Self; + + #[inline] + fn add(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner + rhs.value; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + +impl ops::Sub for Instant { + type Output = Self; + + #[inline] + fn sub(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner - rhs.value; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::Nsec {} + impl Sealed for super::Jiffy {} +} + +/// A trait for time units. +pub trait TimeUnit: private::Sealed { + /// The underlying representation of the time unit. + type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; +} + +/// A time unit of nanoseconds. +/// +/// A [`Delta`] stores its value as [`i64`] nanoseconds and can represent +/// any [`i64`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Nsec {} + +impl TimeUnit for Nsec { + type Repr = i64; +} + +/// A time unit of jiffies. +/// +/// A [`Delta`] stores its value as [`isize`] jiffies and can represent +/// any [`isize`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Jiffy {} + +impl TimeUnit for Jiffy { + type Repr = isize; +} + +/// A span of time. +/// +/// The span is stored in the unit given by the type parameter `U` (see +/// [`TimeUnit`]); its value has type `U::Repr`. `U` defaults to [`Nsec`], so a +/// plain [`Delta`] is a span in nanoseconds. The value can be negative, zero, or +/// positive. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub struct Delta { + value: U::Repr, +} + +impl Delta { + /// Create a new [`Delta`] from a number of jiffies. + #[inline] + pub const fn from_jiffies(jiffies: isize) -> Self { + Self { value: jiffies } + } + + /// Return the number of jiffies in the [`Delta`]. + #[inline] + pub const fn as_jiffies(self) -> isize { + self.value + } +} + +impl ops::Add for Delta { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> Self { + Self { + value: self.value + rhs.value, + } + } +} + +impl ops::AddAssign for Delta { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.value += rhs.value; + } +} + +impl ops::Sub for Delta { + type Output = Self; + + #[inline] + fn sub(self, rhs: Self) -> Self::Output { + Self { + value: self.value - rhs.value, + } + } +} + +impl ops::SubAssign for Delta { + #[inline] + fn sub_assign(&mut self, rhs: Self) { + self.value -= rhs.value; + } +} + +impl ops::Mul for Delta { + type Output = Self; + + #[inline] + fn mul(self, rhs: i64) -> Self::Output { + Self { + value: self.value * rhs, + } + } +} + +impl ops::MulAssign for Delta { + #[inline] + fn mul_assign(&mut self, rhs: i64) { + self.value *= rhs; + } +} + +impl ops::Div for Delta { + type Output = i64; + + #[inline] + fn div(self, rhs: Self) -> Self::Output { + #[cfg(CONFIG_64BIT)] + { + self.value / rhs.value + } + + #[cfg(not(CONFIG_64BIT))] + { + // SAFETY: This function is always safe to call regardless of the input values + unsafe { bindings::div64_s64(self.value, rhs.value) } + } + } +} + +impl Delta { + /// A span of time equal to zero. + pub const ZERO: Self = Self { value: 0 }; + + /// Create a new [`Delta`] from a number of nanoseconds. + #[inline] + pub const fn from_nanos(nanos: i64) -> Self { + Self { value: nanos } + } + + /// Create a new [`Delta`] from a number of microseconds. + /// + /// The `micros` can range from -9_223_372_036_854_775 to 9_223_372_036_854_775. + /// If `micros` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_micros(micros: i64) -> Self { + Self { + value: micros.saturating_mul(NSEC_PER_USEC), + } + } + + /// Create a new [`Delta`] from a number of milliseconds. + /// + /// The `millis` can range from -9_223_372_036_854 to 9_223_372_036_854. + /// If `millis` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_millis(millis: i64) -> Self { + Self { + value: millis.saturating_mul(NSEC_PER_MSEC), + } + } + + /// Create a new [`Delta`] from a number of seconds. + /// + /// The `secs` can range from -9_223_372_036 to 9_223_372_036. + /// If `secs` is outside this range, `i64::MIN` is used for negative values, + /// and `i64::MAX` is used for positive values due to saturation. + #[inline] + pub const fn from_secs(secs: i64) -> Self { + Self { + value: secs.saturating_mul(NSEC_PER_SEC), + } + } + + /// Return `true` if the [`Delta`] spans no time. + #[inline] + pub fn is_zero(self) -> bool { + self.as_nanos() == 0 + } + + /// Return `true` if the [`Delta`] spans a negative amount of time. + #[inline] + pub fn is_negative(self) -> bool { + self.as_nanos() < 0 + } + + /// Return the number of nanoseconds in the [`Delta`]. + #[inline] + pub const fn as_nanos(self) -> i64 { + self.value + } + + /// Return the smallest number of microseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_USEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_us()` with any value. + unsafe { + bindings::ktime_to_us(n) + add + } + } + + /// Return the number of milliseconds in the [`Delta`]. + #[inline] + pub fn as_millis(self) -> i64 { + #[cfg(CONFIG_64BIT)] + { + self.as_nanos() / NSEC_PER_MSEC + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(self.as_nanos()) + } + } + + /// Return the smallest number of milliseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_millis_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_MSEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(n) + add + } + } + + /// Return `self % dividend` where `dividend` is in nanoseconds. + /// + /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is + /// limited to 32 bit dividends. + #[inline] + pub fn rem_nanos(self, dividend: i32) -> Self { + #[cfg(CONFIG_64BIT)] + { + Self { + value: self.as_nanos() % i64::from(dividend), + } + } + + #[cfg(not(CONFIG_64BIT))] + { + let mut rem = 0; + + // SAFETY: `rem` is in the stack, so we can always provide a valid pointer to it. + unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; + + Self { + value: i64::from(rem), + } + } + } +} diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs new file mode 100644 index 000000000..b5b1b4279 --- /dev/null +++ b/rust/kernel/time/delay.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Delay and sleep primitives. +//! +//! This module contains the kernel APIs related to delay and sleep that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h). + +use super::Delta; +use crate::prelude::*; + +/// Sleeps for a given duration at least. +/// +/// Equivalent to the C side [`fsleep()`], flexible sleep function, +/// which automatically chooses the best sleep method based on a duration. +/// +/// `delta` must be within `[0, i32::MAX]` microseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug +/// to call this function with an out-of-range value, in which case the function +/// will sleep for at least the maximum value in the range and may warn +/// in the future. +/// +/// The behavior above differs from the C side [`fsleep()`] for which out-of-range +/// values mean "infinite timeout" instead. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`fsleep()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.fsleep +pub fn fsleep(delta: Delta) { + // The maximum value is set to `i32::MAX` microseconds to prevent integer + // overflow inside fsleep, which could lead to unintentional infinite sleep. + const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64); + + let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) { + delta + } else { + // TODO: Add WARN_ONCE() when it's supported. + MAX_DELTA + }; + + // SAFETY: It is always safe to call `fsleep()` with any duration. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `fsleep()` sleeps for at least the provided duration, + // but that it may sleep for longer under some circumstances. + bindings::fsleep(delta.as_micros_ceil() as c_ulong) + } +} + +/// Inserts a delay based on microseconds with busy waiting. +/// +/// Equivalent to the C side [`udelay()`], which delays in microseconds. +/// +/// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug to +/// call this function with an out-of-range value. +/// +/// The behavior above differs from the C side [`udelay()`] for which out-of-range +/// values could lead to an overflow and unexpected behavior. +/// +/// [`udelay()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.udelay +pub fn udelay(delta: Delta) { + const MAX_UDELAY_DELTA: Delta = Delta::from_millis(bindings::MAX_UDELAY_MS as i64); + + debug_assert!(delta.as_nanos() >= 0); + debug_assert!(delta <= MAX_UDELAY_DELTA); + + let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) { + delta + } else { + MAX_UDELAY_DELTA + }; + + // SAFETY: It is always safe to call `udelay()` with any duration. + // Note that the kernel is compiled with `-fno-strict-overflow` + // so any out-of-range value could lead to unexpected behavior + // but won't lead to undefined behavior. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `udelay()` inserts a delay for at least + // the provided duration, but that it may delay for longer + // under some circumstances. + bindings::udelay(delta.as_micros_ceil() as c_ulong) + } +} diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs new file mode 100644 index 000000000..2d7f1131a --- /dev/null +++ b/rust/kernel/time/hrtimer.rs @@ -0,0 +1,1120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Intrusive high resolution timers. +//! +//! Allows running timer callbacks without doing allocations at the time of +//! starting the timer. For now, only one timer per type is allowed. +//! +//! # Vocabulary +//! +//! States: +//! +//! - Stopped: initialized but not started, or cancelled, or not restarted. +//! - Started: initialized and started or restarted. +//! - Running: executing the callback. +//! +//! Operations: +//! +//! * Start +//! * Cancel +//! * Restart +//! +//! Events: +//! +//! * Expire +//! +//! ## State Diagram +//! +//! ```text +//! Return NoRestart +//! +---------------------------------------------------------------------+ +//! | | +//! | | +//! | | +//! | Return Restart | +//! | +------------------------+ | +//! | | | | +//! | | | | +//! v v | | +//! +-----------------+ Start +------------------+ +--------+-----+--+ +//! | +---------------->| | | | +//! Init | | | | Expire | | +//! --------->| Stopped | | Started +---------->| Running | +//! | | Cancel | | | | +//! | |<----------------+ | | | +//! +-----------------+ +---------------+--+ +-----------------+ +//! ^ | +//! | | +//! +---------+ +//! Restart +//! ``` +//! +//! +//! A timer is initialized in the **stopped** state. A stopped timer can be +//! **started** by the `start` operation, with an **expiry** time. After the +//! `start` operation, the timer is in the **started** state. When the timer +//! **expires**, the timer enters the **running** state and the handler is +//! executed. After the handler has returned, the timer may enter the +//! **started* or **stopped** state, depending on the return value of the +//! handler. A timer in the **started** or **running** state may be **canceled** +//! by the `cancel` operation. A timer that is cancelled enters the **stopped** +//! state. +//! +//! A `cancel` or `restart` operation on a timer in the **running** state takes +//! effect after the handler has returned and the timer has transitioned +//! out of the **running** state. +//! +//! A `restart` operation on a timer in the **stopped** state is equivalent to a +//! `start` operation. +//! +//! When a type implements both `HrTimerPointer` and `Clone`, it is possible to +//! issue the `start` operation while the timer is in the **started** state. In +//! this case the `start` operation is equivalent to the `restart` operation. +//! +//! # Examples +//! +//! ## Using an intrusive timer living in a [`Box`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, +//! # HrTimerRestart, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct BoxIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! shared: Arc, +//! } +//! +//! impl BoxIntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for BoxIntrusiveHrTimer { +//! type Pointer<'a> = Pin>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for BoxIntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! let has_timer = Box::pin_init(BoxIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let shared = has_timer.shared.clone(); +//! let _handle = has_timer.start(Delta::from_micros(200)); +//! +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using an intrusive timer in an [`Arc`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, ArcBorrow, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct ArcIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl ArcIntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for ArcIntrusiveHrTimer { +//! type Pointer<'a> = Arc; +//! +//! fn run( +//! this: ArcBorrow<'_, Self>, +//! _ctx: HrTimerCallbackContext<'_, Self>, +//! ) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.flag.fetch_add(1, ordering::Full); +//! this.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for ArcIntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! let has_timer = Arc::pin_init(ArcIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let _handle = has_timer.clone().start(Delta::from_micros(200)); +//! +//! while has_timer.flag.load(ordering::Relaxed) != 5 { +//! has_timer.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_pin_init; +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a Self>; +//! +//! fn run(this: Pin<&Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! this.flag.store(1, ordering::Release); +//! this.cond.complete_all(); +//! +//! HrTimerRestart::NoRestart +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for IntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! stack_pin_init!( let has_timer = IntrusiveHrTimer::new() ); +//! has_timer.as_ref().start_scoped(Delta::from_micros(200), || { +//! while has_timer.flag.load(ordering::Relaxed) != 1 { +//! has_timer.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Flag raised\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a mutable stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_try_pin_init; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer, +//! shared: Arc, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a mut Self>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer for IntrusiveHrTimer { +//! mode: RelativeMode, field: self.timer +//! } +//! } +//! +//! stack_try_pin_init!( let has_timer =? IntrusiveHrTimer::new() ); +//! let shared = has_timer.shared.clone(); +//! +//! has_timer.as_mut().start_scoped(Delta::from_micros(200), || { +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! [`Arc`]: kernel::sync::Arc + +use super::{ClockSource, Delta, Instant}; +use crate::{prelude::*, types::Opaque}; +use core::{marker::PhantomData, ptr::NonNull}; +use pin_init::PinInit; + +/// A type-alias to refer to the [`Instant`] for a given `T` from [`HrTimer`]. +/// +/// Where `C` is the [`ClockSource`] of the [`HrTimer`]. +pub type HrTimerInstant = Instant<<>::TimerMode as HrTimerMode>::Clock>; + +/// A timer backed by a C `struct hrtimer`. +/// +/// # Invariants +/// +/// * `self.timer` is initialized by `bindings::hrtimer_setup`. +#[pin_data] +#[repr(C)] +pub struct HrTimer { + #[pin] + timer: Opaque, + _t: PhantomData, +} + +// SAFETY: Ownership of an `HrTimer` can be moved to other threads and +// used/dropped from there. +unsafe impl Send for HrTimer {} + +// SAFETY: Timer operations are locked on the C side, so it is safe to operate +// on a timer from multiple threads. +unsafe impl Sync for HrTimer {} + +impl HrTimer { + /// Return an initializer for a new timer instance. + pub fn new() -> impl PinInit + where + T: HrTimerCallback, + T: HasHrTimer, + { + pin_init!(Self { + // INVARIANT: We initialize `timer` with `hrtimer_setup` below. + timer <- Opaque::ffi_init(move |place: *mut bindings::hrtimer| { + // SAFETY: By design of `pin_init!`, `place` is a pointer to a + // live allocation. hrtimer_setup will initialize `place` and + // does not require `place` to be initialized prior to the call. + unsafe { + bindings::hrtimer_setup( + place, + Some(T::Pointer::run), + <>::TimerMode as HrTimerMode>::Clock::ID, + >::TimerMode::C_MODE, + ); + } + }), + _t: PhantomData, + }) + } + + /// Get a pointer to the contained `bindings::hrtimer`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a live allocation of at least the size of `Self`. + unsafe fn raw_get(this: *const Self) -> *mut bindings::hrtimer { + // SAFETY: The field projection to `timer` does not go out of bounds, + // because the caller of this function promises that `this` points to an + // allocation of at least the size of `Self`. + unsafe { Opaque::cast_into(core::ptr::addr_of!((*this).timer)) } + } + + /// Cancel an initialized and potentially running timer. + /// + /// If the timer handler is running, this function will block until the + /// handler returns. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + /// + /// Users of the `HrTimer` API would not usually call this method directly. + /// Instead they would use the safe [`HrTimerHandle::cancel`] on the handle + /// returned when the timer was started. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a valid `Self`. + pub(crate) unsafe fn raw_cancel(this: *const Self) -> bool { + // SAFETY: `this` points to an allocation of at least `HrTimer` size. + let c_timer_ptr = unsafe { HrTimer::raw_get(this) }; + + // If the handler is running, this will wait for the handler to return + // before returning. + // SAFETY: `c_timer_ptr` is initialized and valid. Synchronization is + // handled on the C side. + unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 } + } + + /// Forward the timer expiry for a given timer pointer. + /// + /// # Safety + /// + /// - `self_ptr` must point to a valid `Self`. + /// - The caller must either have exclusive access to the data pointed at by `self_ptr`, or be + /// within the context of the timer callback. + #[inline] + unsafe fn raw_forward(self_ptr: *mut Self, now: HrTimerInstant, interval: Delta) -> u64 + where + T: HasHrTimer, + { + // SAFETY: + // * The C API requirements for this function are fulfilled by our safety contract. + // * `self_ptr` is guaranteed to point to a valid `Self` via our safety contract + unsafe { + bindings::hrtimer_forward(Self::raw_get(self_ptr), now.as_nanos(), interval.as_nanos()) + } + } + + /// Conditionally forward the timer. + /// + /// If the timer expires after `now`, this function does nothing and returns 0. If the timer + /// expired at or before `now`, this function forwards the timer by `interval` until the timer + /// expires after `now` and then returns the number of times the timer was forwarded by + /// `interval`. + /// + /// This function is mainly useful for timer types which can provide exclusive access to the + /// timer when the timer is not running. For forwarding the timer from within the timer callback + /// context, see [`HrTimerCallbackContext::forward()`]. + /// + /// Returns the number of overruns that occurred as a result of the timer expiry change. + pub fn forward(self: Pin<&mut Self>, now: HrTimerInstant, interval: Delta) -> u64 + where + T: HasHrTimer, + { + // SAFETY: `raw_forward` does not move `Self` + let this = unsafe { self.get_unchecked_mut() }; + + // SAFETY: By existence of `Pin<&mut Self>`, the pointer passed to `raw_forward` points to a + // valid `Self` that we have exclusive access to. + unsafe { Self::raw_forward(this, now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`forward()`](Self::forward) that uses an interval after the current + /// time of the base clock for the [`HrTimer`]. + pub fn forward_now(self: Pin<&mut Self>, interval: Delta) -> u64 + where + T: HasHrTimer, + { + self.forward(HrTimerInstant::::now(), interval) + } + + /// Return the time expiry for this [`HrTimer`]. + /// + /// This value should only be used as a snapshot, as the actual expiry time could change after + /// this function is called. + pub fn expires(&self) -> HrTimerInstant + where + T: HasHrTimer, + { + // SAFETY: `self` is an immutable reference and thus always points to a valid `HrTimer`. + let c_timer_ptr = unsafe { HrTimer::raw_get(self) }; + + // SAFETY: + // - Timers cannot have negative ktime_t values as their expiration time. + // - There's no actual locking here, a racy read is fine and expected + unsafe { + Instant::from_ktime( + // This `read_volatile` is intended to correspond to a READ_ONCE call. + // FIXME(read_once): Replace with `read_once` when available on the Rust side. + core::ptr::read_volatile(&raw const ((*c_timer_ptr).node.expires)), + ) + } + } +} + +/// Implemented by pointer types that point to structs that contain a [`HrTimer`]. +/// +/// `Self` must be [`Sync`] because it is passed to timer callbacks in another +/// thread of execution (hard or soft interrupt context). +/// +/// Starting a timer returns a [`HrTimerHandle`] that can be used to manipulate +/// the timer. Note that it is OK to call the start function repeatedly, and +/// that more than one [`HrTimerHandle`] associated with a [`HrTimerPointer`] may +/// exist. A timer can be manipulated through any of the handles, and a handle +/// may represent a cancelled timer. +pub trait HrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// A handle representing a started or restarted timer. + /// + /// If the timer is running or if the timer callback is executing when the + /// handle is dropped, the drop method of [`HrTimerHandle`] should not return + /// until the timer is stopped and the callback has completed. + /// + /// Note: When implementing this trait, consider that it is not unsafe to + /// leak the handle. + type TimerHandle: HrTimerHandle; + + /// Start the timer with expiry after `expires` time units. If the timer was + /// already running, it is restarted with the new expiry time. + fn start(self, expires: ::Expires) -> Self::TimerHandle; +} + +/// Unsafe version of [`HrTimerPointer`] for situations where leaking the +/// [`HrTimerHandle`] returned by `start` would be unsound. This is the case for +/// stack allocated timers. +/// +/// Typical implementers are pinned references such as [`Pin<&T>`]. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that instances of types implementing +/// [`UnsafeHrTimerPointer`] outlives any associated [`HrTimerPointer::TimerHandle`] +/// instances. +pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// A handle representing a running timer. + /// + /// # Safety + /// + /// If the timer is running, or if the timer callback is executing when the + /// handle is dropped, the drop method of [`Self::TimerHandle`] must not return + /// until the timer is stopped and the callback has completed. + type TimerHandle: HrTimerHandle; + + /// Start the timer after `expires` time units. If the timer was already + /// running, it is restarted at the new expiry time. + /// + /// # Safety + /// + /// Caller promises keep the timer structure alive until the timer is dead. + /// Caller can ensure this by not leaking the returned [`Self::TimerHandle`]. + unsafe fn start(self, expires: ::Expires) -> Self::TimerHandle; +} + +/// A trait for stack allocated timers. +/// +/// # Safety +/// +/// Implementers must ensure that `start_scoped` does not return until the +/// timer is dead and the timer handler is not running. +pub unsafe trait ScopedHrTimerPointer { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// Start the timer to run after `expires` time units and immediately + /// after call `f`. When `f` returns, the timer is cancelled. + fn start_scoped(self, expires: ::Expires, f: F) -> T + where + F: FnOnce() -> T; +} + +// SAFETY: By the safety requirement of [`UnsafeHrTimerPointer`], dropping the +// handle returned by [`UnsafeHrTimerPointer::start`] ensures that the timer is +// killed. +unsafe impl ScopedHrTimerPointer for T +where + T: UnsafeHrTimerPointer, +{ + type TimerMode = T::TimerMode; + + fn start_scoped( + self, + expires: <::TimerMode as HrTimerMode>::Expires, + f: F, + ) -> U + where + F: FnOnce() -> U, + { + // SAFETY: We drop the timer handle below before returning. + let handle = unsafe { UnsafeHrTimerPointer::start(self, expires) }; + let t = f(); + drop(handle); + t + } +} + +/// Implemented by [`HrTimerPointer`] implementers to give the C timer callback a +/// function to call. +// This is split from `HrTimerPointer` to make it easier to specify trait bounds. +pub trait RawHrTimerCallback { + /// Type of the parameter passed to [`HrTimerCallback::run`]. It may be + /// [`Self`], or a pointer type derived from [`Self`]. + type CallbackTarget<'a>; + + /// Callback to be called from C when timer fires. + /// + /// # Safety + /// + /// Only to be called by C code in the `hrtimer` subsystem. `this` must point + /// to the `bindings::hrtimer` structure that was used to start the timer. + unsafe extern "C" fn run(this: *mut bindings::hrtimer) -> bindings::hrtimer_restart; +} + +/// Implemented by structs that can be the target of a timer callback. +pub trait HrTimerCallback { + /// The type whose [`RawHrTimerCallback::run`] method will be invoked when + /// the timer expires. + type Pointer<'a>: RawHrTimerCallback; + + /// Called by the timer logic when the timer fires. + fn run( + this: as RawHrTimerCallback>::CallbackTarget<'_>, + ctx: HrTimerCallbackContext<'_, Self>, + ) -> HrTimerRestart + where + Self: Sized, + Self: HasHrTimer; +} + +/// A handle representing a potentially running timer. +/// +/// More than one handle representing the same timer might exist. +/// +/// # Safety +/// +/// When dropped, the timer represented by this handle must be cancelled, if it +/// is running. If the timer handler is running when the handle is dropped, the +/// drop method must wait for the handler to return before returning. +/// +/// Note: One way to satisfy the safety requirement is to call `Self::cancel` in +/// the drop implementation for `Self.` +pub unsafe trait HrTimerHandle { + /// Cancel the timer. If the timer is in the running state, block till the + /// handler has returned. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + /// + /// Returns `true` if the timer was running. + fn cancel(&mut self) -> bool; +} + +/// Implemented by structs that contain timer nodes. +/// +/// Clients of the timer API would usually safely implement this trait by using +/// the [`crate::impl_has_hr_timer`] macro. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that the implementer has a +/// [`HrTimer`] field and that all trait methods are implemented according to +/// their documentation. All the methods of this trait must operate on the same +/// field. +pub unsafe trait HasHrTimer { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + + /// Return a pointer to the [`HrTimer`] within `Self`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn raw_get_timer(this: *const Self) -> *const HrTimer; + + /// Return a pointer to the struct that is containing the [`HrTimer`] pointed + /// to by `ptr`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `ptr` must point to a [`HrTimer`] field in a struct of type `Self`. + unsafe fn timer_container_of(ptr: *mut HrTimer) -> *mut Self + where + Self: Sized; + + /// Get pointer to the contained `bindings::hrtimer` struct. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn c_timer_ptr(this: *const Self) -> *const bindings::hrtimer { + // SAFETY: `this` is a valid pointer to a `Self`. + let timer_ptr = unsafe { Self::raw_get_timer(this) }; + + // SAFETY: timer_ptr points to an allocation of at least `HrTimer` size. + unsafe { HrTimer::raw_get(timer_ptr) } + } + + /// Start the timer contained in the `Self` pointed to by `self_ptr`. If + /// it is already running it is removed and inserted. + /// + /// # Safety + /// + /// - `this` must point to a valid `Self`. + /// - Caller must ensure that the pointee of `this` lives until the timer + /// fires or is canceled. + unsafe fn start(this: *const Self, expires: ::Expires) { + // SAFETY: By function safety requirement, `this` is a valid `Self`. + unsafe { + bindings::hrtimer_start_range_ns( + Self::c_timer_ptr(this).cast_mut(), + expires.as_nanos(), + 0, + ::C_MODE, + ); + } + } +} + +/// Restart policy for timers. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum HrTimerRestart { + /// Timer should not be restarted. + NoRestart = bindings::hrtimer_restart_HRTIMER_NORESTART, + /// Timer should be restarted. + Restart = bindings::hrtimer_restart_HRTIMER_RESTART, +} + +impl HrTimerRestart { + fn into_c(self) -> bindings::hrtimer_restart { + self as bindings::hrtimer_restart + } +} + +/// Time representations that can be used as expiration values in [`HrTimer`]. +pub trait HrTimerExpires { + /// Converts the expiration time into a nanosecond representation. + /// + /// This value corresponds to a raw ktime_t value, suitable for passing to kernel + /// timer functions. The interpretation (absolute vs relative) depends on the + /// associated [HrTimerMode] in use. + fn as_nanos(&self) -> i64; +} + +impl HrTimerExpires for Instant { + #[inline] + fn as_nanos(&self) -> i64 { + Instant::::as_nanos(self) + } +} + +impl HrTimerExpires for Delta { + #[inline] + fn as_nanos(&self) -> i64 { + Delta::as_nanos(*self) + } +} + +mod private { + use crate::time::ClockSource; + + pub trait Sealed {} + + impl Sealed for super::AbsoluteMode {} + impl Sealed for super::RelativeMode {} + impl Sealed for super::AbsolutePinnedMode {} + impl Sealed for super::RelativePinnedMode {} + impl Sealed for super::AbsoluteSoftMode {} + impl Sealed for super::RelativeSoftMode {} + impl Sealed for super::AbsolutePinnedSoftMode {} + impl Sealed for super::RelativePinnedSoftMode {} + impl Sealed for super::AbsoluteHardMode {} + impl Sealed for super::RelativeHardMode {} + impl Sealed for super::AbsolutePinnedHardMode {} + impl Sealed for super::RelativePinnedHardMode {} +} + +/// Operational mode of [`HrTimer`]. +pub trait HrTimerMode: private::Sealed { + /// The C representation of hrtimer mode. + const C_MODE: bindings::hrtimer_mode; + + /// Type representing the clock source. + type Clock: ClockSource; + + /// Type representing the expiration specification (absolute or relative time). + type Expires: HrTimerExpires; +} + +/// Timer that expires at a fixed point in time. +pub struct AbsoluteMode(PhantomData); + +impl HrTimerMode for AbsoluteMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS; + + type Clock = C; + type Expires = Instant; +} + +/// Timer that expires after a delay from now. +pub struct RelativeMode(PhantomData); + +impl HrTimerMode for RelativeMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration time, pinned to its current CPU. +pub struct AbsolutePinnedMode(PhantomData); +impl HrTimerMode for AbsolutePinnedMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration time, pinned to its current CPU. +pub struct RelativePinnedMode(PhantomData); +impl HrTimerMode for RelativePinnedMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in soft irq context. +pub struct AbsoluteSoftMode(PhantomData); +impl HrTimerMode for AbsoluteSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, handled in soft irq context. +pub struct RelativeSoftMode(PhantomData); +impl HrTimerMode for RelativeSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct AbsolutePinnedSoftMode(PhantomData); +impl HrTimerMode for AbsolutePinnedSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct RelativePinnedSoftMode(PhantomData); +impl HrTimerMode for RelativePinnedSoftMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in hard irq context. +pub struct AbsoluteHardMode(PhantomData); +impl HrTimerMode for AbsoluteHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, handled in hard irq context. +pub struct RelativeHardMode(PhantomData); +impl HrTimerMode for RelativeHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in hard irq context. +pub struct AbsolutePinnedHardMode(PhantomData); +impl HrTimerMode for AbsolutePinnedHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD; + + type Clock = C; + type Expires = Instant; +} + +/// Timer with relative expiration, pinned to CPU and handled in hard irq context. +pub struct RelativePinnedHardMode(PhantomData); +impl HrTimerMode for RelativePinnedHardMode { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD; + + type Clock = C; + type Expires = Delta; +} + +/// Privileged smart-pointer for a [`HrTimer`] callback context. +/// +/// Many [`HrTimer`] methods can only be called in two situations: +/// +/// * When the caller has exclusive access to the `HrTimer` and the `HrTimer` is guaranteed not to +/// be running. +/// * From within the context of an `HrTimer`'s callback method. +/// +/// This type provides access to said methods from within a timer callback context. +/// +/// # Invariants +/// +/// * The existence of this type means the caller is currently within the callback for an +/// [`HrTimer`]. +/// * `self.0` always points to a live instance of [`HrTimer`]. +pub struct HrTimerCallbackContext<'a, T: HasHrTimer>(NonNull>, PhantomData<&'a ()>); + +impl<'a, T: HasHrTimer> HrTimerCallbackContext<'a, T> { + /// Create a new [`HrTimerCallbackContext`]. + /// + /// # Safety + /// + /// This function relies on the caller being within the context of a timer callback, so it must + /// not be used anywhere except for within implementations of [`RawHrTimerCallback::run`]. The + /// caller promises that `timer` points to a valid initialized instance of + /// [`bindings::hrtimer`]. + /// + /// The returned `Self` must not outlive the function context of [`RawHrTimerCallback::run`] + /// where this function is called. + pub(crate) unsafe fn from_raw(timer: *mut HrTimer) -> Self { + // SAFETY: The caller guarantees `timer` is a valid pointer to an initialized + // `bindings::hrtimer` + // INVARIANT: Our safety contract ensures that we're within the context of a timer callback + // and that `timer` points to a live instance of `HrTimer`. + Self(unsafe { NonNull::new_unchecked(timer) }, PhantomData) + } + + /// Conditionally forward the timer. + /// + /// This function is identical to [`HrTimer::forward()`] except that it may only be used from + /// within the context of a [`HrTimer`] callback. + pub fn forward(&mut self, now: HrTimerInstant, interval: Delta) -> u64 { + // SAFETY: + // - We are guaranteed to be within the context of a timer callback by our type invariants + // - By our type invariants, `self.0` always points to a valid `HrTimer` + unsafe { HrTimer::::raw_forward(self.0.as_ptr(), now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`HrTimerCallbackContext::forward()`] that uses an interval after the + /// current time of the base clock for the [`HrTimer`]. + pub fn forward_now(&mut self, duration: Delta) -> u64 { + self.forward(HrTimerInstant::::now(), duration) + } +} + +/// Use to implement the [`HasHrTimer`] trait. +/// +/// See [`module`] documentation for an example. +/// +/// [`module`]: crate::time::hrtimer +#[macro_export] +macro_rules! impl_has_hr_timer { + ( + impl$({$($generics:tt)*})? + HasHrTimer<$timer_type:ty> + for $self:ty + { + mode : $mode:ty, + field : self.$field:ident $(,)? + } + $($rest:tt)* + ) => { + // SAFETY: This implementation of `raw_get_timer` only compiles if the + // field has the right type. + unsafe impl$(<$($generics)*>)? $crate::time::hrtimer::HasHrTimer<$timer_type> for $self { + type TimerMode = $mode; + + #[inline] + unsafe fn raw_get_timer( + this: *const Self, + ) -> *const $crate::time::hrtimer::HrTimer<$timer_type> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { ::core::ptr::addr_of!((*this).$field) } + } + + #[inline] + unsafe fn timer_container_of( + ptr: *mut $crate::time::hrtimer::HrTimer<$timer_type>, + ) -> *mut Self { + // SAFETY: As per the safety requirement of this function, `ptr` + // is pointing inside a `$timer_type`. + unsafe { ::kernel::container_of!(ptr, $timer_type, $field) } + } + } + } +} + +mod arc; +pub use arc::ArcHrTimerHandle; +mod pin; +pub use pin::PinHrTimerHandle; +mod pin_mut; +pub use pin_mut::PinMutHrTimerHandle; +// `box` is a reserved keyword, so prefix with `t` for timer +mod tbox; +pub use tbox::BoxHrTimerHandle; diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs new file mode 100644 index 000000000..7be82bcb3 --- /dev/null +++ b/rust/kernel/time/hrtimer/arc.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::sync::Arc; +use crate::sync::ArcBorrow; + +/// A handle for an `Arc>` returned by a call to +/// [`HrTimerPointer::start`]. +pub struct ArcHrTimerHandle +where + T: HasHrTimer, +{ + pub(crate) inner: Arc, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl HrTimerHandle for ArcHrTimerHandle +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr = Arc::as_ptr(&self.inner); + + // SAFETY: As we obtained `self_ptr` from a valid reference above, it + // must point to a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl Drop for ArcHrTimerHandle +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +impl HrTimerPointer for Arc +where + T: 'static, + T: Send + Sync, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = ArcHrTimerHandle; + + fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> ArcHrTimerHandle { + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(Arc::as_ptr(&self), expires) }; + ArcHrTimerHandle { inner: self } + } +} + +impl RawHrTimerCallback for Arc +where + T: 'static, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Self>, +{ + type CallbackTarget<'a> = ArcBorrow<'a, T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - `data_ptr` is derived form the pointer to the `T` that was used to + // queue the timer. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `ArcHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the `ArcBorrow` created below. + // - We own one refcount in the `ArcTimerHandle` associated with this + // timer, so it is not possible to get a `UniqueArc` to this + // allocation from other `Arc` clones. + let receiver = unsafe { ArcBorrow::from_raw(data_ptr) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs new file mode 100644 index 000000000..4d39ef781 --- /dev/null +++ b/rust/kernel/time/hrtimer/pin.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::RawHrTimerCallback; +use super::UnsafeHrTimerPointer; +use core::pin::Pin; + +/// A handle for a `Pin<&HasHrTimer>`. When the handle exists, the timer might be +/// running. +pub struct PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + pub(crate) inner: Pin<&'a T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr: *const T = self.inner.get_ref(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a `PinHrTimerHandle`, +// so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a T> +where + T: Send + Sync, + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = PinHrTimerHandle<'a, T>; + + unsafe fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // Cast to pointer + let self_ptr: *const T = self.get_ref(); + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr, expires) }; + + PinHrTimerHandle { inner: self } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a T> +where + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr`, thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &*receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs new file mode 100644 index 000000000..9d9447d4d --- /dev/null +++ b/rust/kernel/time/hrtimer/pin_mut.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::{ + HasHrTimer, HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerHandle, HrTimerMode, + RawHrTimerCallback, UnsafeHrTimerPointer, +}; +use core::{marker::PhantomData, pin::Pin, ptr::NonNull}; + +/// A handle for a `Pin<&mut HasHrTimer>`. When the handle exists, the timer might +/// be running. +pub struct PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + pub(crate) inner: NonNull, + _p: PhantomData<&'a mut T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn cancel(&mut self) -> bool { + let self_ptr = self.inner.as_ptr(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a +// `PinMutHrTimerHandle`, so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a mut T> +where + T: Send + Sync, + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type TimerMode = >::TimerMode; + type TimerHandle = PinMutHrTimerHandle<'a, T>; + + unsafe fn start( + mut self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // SAFETY: + // - We promise not to move out of `self`. We only pass `self` + // back to the caller as a `Pin<&mut self>`. + // - The return value of `get_unchecked_mut` is guaranteed not to be null. + let self_ptr = unsafe { NonNull::new_unchecked(self.as_mut().get_unchecked_mut()) }; + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr.as_ptr(), expires) }; + + PinMutHrTimerHandle { + inner: self_ptr, + _p: PhantomData, + } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a mut T> +where + T: HasHrTimer, + T: HrTimerCallback = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinMutHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr` mutably thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &mut *receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs new file mode 100644 index 000000000..aa1ee31a7 --- /dev/null +++ b/rust/kernel/time/hrtimer/tbox.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerCallbackContext; +use super::HrTimerHandle; +use super::HrTimerMode; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::prelude::*; +use core::ptr::NonNull; + +/// A handle for a [`Box>`] returned by a call to +/// [`HrTimerPointer::start`]. +/// +/// # Invariants +/// +/// - `self.inner` comes from a `Box::into_raw` call. +pub struct BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + pub(crate) inner: NonNull, + _p: core::marker::PhantomData, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl HrTimerHandle for BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + fn cancel(&mut self) -> bool { + // SAFETY: As we obtained `self.inner` from a valid reference when we + // created `self`, it must point to a valid `T`. + let timer_ptr = unsafe { >::raw_get_timer(self.inner.as_ptr()) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::::raw_cancel(timer_ptr) } + } +} + +impl Drop for BoxHrTimerHandle +where + T: HasHrTimer, + A: crate::alloc::Allocator, +{ + fn drop(&mut self) { + self.cancel(); + // SAFETY: By type invariant, `self.inner` came from a `Box::into_raw` + // call. + drop(unsafe { Box::::from_raw(self.inner.as_ptr()) }) + } +} + +impl HrTimerPointer for Pin> +where + T: 'static, + T: Send + Sync, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Pin>>, + A: crate::alloc::Allocator, +{ + type TimerMode = >::TimerMode; + type TimerHandle = BoxHrTimerHandle; + + fn start( + self, + expires: <>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { + // SAFETY: + // - We will not move out of this box during timer callback (we pass an + // immutable reference to the callback). + // - `Box::into_raw` is guaranteed to return a valid pointer. + let inner = + unsafe { NonNull::new_unchecked(Box::into_raw(Pin::into_inner_unchecked(self))) }; + + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(inner.as_ptr(), expires) }; + + // INVARIANT: `inner` came from `Box::into_raw` above. + BoxHrTimerHandle { + inner, + _p: core::marker::PhantomData, + } + } +} + +impl RawHrTimerCallback for Pin> +where + T: 'static, + T: HasHrTimer, + T: for<'a> HrTimerCallback = Pin>>, + A: crate::alloc::Allocator, +{ + type CallbackTarget<'a> = Pin<&'a mut T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `BoxHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle owns the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the reference created below. + // - As `data_ptr` comes from a `Pin>`, only pinned references to + // `data_ptr` exist. + let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) }; + + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(data_mut_ref, context).into_c() + } +} diff --git a/rust/kernel/tracepoint.rs b/rust/kernel/tracepoint.rs new file mode 100644 index 000000000..c6e80aa99 --- /dev/null +++ b/rust/kernel/tracepoint.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for tracepoints. + +/// Declare the Rust entry point for a tracepoint. +/// +/// This macro generates an unsafe function that calls into C, and its safety requirements will be +/// whatever the relevant C code requires. To document these safety requirements, you may add +/// doc-comments when invoking the macro. +#[macro_export] +macro_rules! declare_trace { + ($($(#[$attr:meta])* $pub:vis unsafe fn $name:ident($($argname:ident : $argtyp:ty),* $(,)?);)*) => {$( + $( #[$attr] )* + #[inline(always)] + $pub unsafe fn $name($($argname : $argtyp),*) { + #[cfg(CONFIG_TRACEPOINTS)] + { + // SAFETY: It's always okay to query the static key for a tracepoint. + let should_trace = unsafe { + $crate::macros::paste! { + $crate::jump_label::static_branch_unlikely!( + $crate::bindings::[< __tracepoint_ $name >], + $crate::bindings::tracepoint, + key + ) + } + }; + + if should_trace { + $crate::macros::paste! { + // SAFETY: The caller guarantees that it is okay to call this tracepoint. + unsafe { $crate::bindings::[< rust_do_trace_ $name >]($($argname),*) }; + } + } + } + + #[cfg(not(CONFIG_TRACEPOINTS))] + { + // If tracepoints are disabled, insert a trivial use of each argument + // to avoid unused argument warnings. + $( let _unused = $argname; )* + } + } + )*} +} + +pub use declare_trace; diff --git a/rust/kernel/transmute.rs b/rust/kernel/transmute.rs new file mode 100644 index 000000000..654b5ede2 --- /dev/null +++ b/rust/kernel/transmute.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Traits for transmuting types. + +use core::mem::size_of; + +/// Types for which any bit pattern is valid. +/// +/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so +/// reading arbitrary bytes into something that contains a `bool` is not okay. +/// +/// It's okay for the type to have padding, as initializing those bytes has no effect. +/// +/// # Examples +/// +/// ``` +/// use kernel::transmute::FromBytes; +/// +/// # fn test() -> Option<()> { +/// let raw = [1, 2, 3, 4]; +/// +/// let result = u32::from_bytes(&raw)?; +/// +/// #[cfg(target_endian = "little")] +/// assert_eq!(*result, 0x4030201); +/// +/// #[cfg(target_endian = "big")] +/// assert_eq!(*result, 0x1020304); +/// +/// # Some(()) } +/// # test().ok_or(EINVAL)?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Safety +/// +/// All bit-patterns must be valid for this type. This type must not have interior mutability. +pub unsafe trait FromBytes { + /// Converts a slice of bytes to a reference to `Self`. + /// + /// Succeeds if the reference is properly aligned, and the size of `bytes` is equal to that of + /// `T` and different from zero. + /// + /// Otherwise, returns [`None`]. + fn from_bytes(bytes: &[u8]) -> Option<&Self> + where + Self: Sized, + { + let slice_ptr = bytes.as_ptr().cast::(); + let size = size_of::(); + + if bytes.len() == size && slice_ptr.is_aligned() { + // SAFETY: Size and alignment were just checked. + unsafe { Some(&*slice_ptr) } + } else { + None + } + } + + /// Converts the beginning of `bytes` to a reference to `Self`. + /// + /// This method is similar to [`Self::from_bytes`], with the difference that `bytes` does not + /// need to be the same size of `Self` - the appropriate portion is cut from the beginning of + /// `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_prefix(bytes: &[u8]) -> Option<(&Self, &[u8])> + where + Self: Sized, + { + let (prefix, remainder) = bytes.split_at_checked(size_of::())?; + + Self::from_bytes(prefix).map(|s| (s, remainder)) + } + + /// Converts a mutable slice of bytes to a reference to `Self`. + /// + /// Succeeds if the reference is properly aligned, and the size of `bytes` is equal to that of + /// `T` and different from zero. + /// + /// Otherwise, returns [`None`]. + fn from_bytes_mut(bytes: &mut [u8]) -> Option<&mut Self> + where + Self: AsBytes + Sized, + { + let slice_ptr = bytes.as_mut_ptr().cast::(); + let size = size_of::(); + + if bytes.len() == size && slice_ptr.is_aligned() { + // SAFETY: Size and alignment were just checked. + unsafe { Some(&mut *slice_ptr) } + } else { + None + } + } + + /// Converts the beginning of `bytes` to a mutable reference to `Self`. + /// + /// This method is similar to [`Self::from_bytes_mut`], with the difference that `bytes` does + /// not need to be the same size of `Self` - the appropriate portion is cut from the beginning + /// of `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_mut_prefix(bytes: &mut [u8]) -> Option<(&mut Self, &mut [u8])> + where + Self: AsBytes + Sized, + { + let (prefix, remainder) = bytes.split_at_mut_checked(size_of::())?; + + Self::from_bytes_mut(prefix).map(|s| (s, remainder)) + } + + /// Creates an owned instance of `Self` by copying `bytes`. + /// + /// Unlike [`FromBytes::from_bytes`], which requires aligned input, this method can be used on + /// non-aligned data at the cost of a copy. + fn from_bytes_copy(bytes: &[u8]) -> Option + where + Self: Sized, + { + if bytes.len() == size_of::() { + // SAFETY: we just verified that `bytes` has the same size as `Self`, and per the + // invariants of `FromBytes`, any byte sequence of the correct length is a valid value + // for `Self`. + Some(unsafe { core::ptr::read_unaligned(bytes.as_ptr().cast::()) }) + } else { + None + } + } + + /// Creates an owned instance of `Self` from the beginning of `bytes`. + /// + /// This method is similar to [`Self::from_bytes_copy`], with the difference that `bytes` does + /// not need to be the same size of `Self` - the appropriate portion is cut from the beginning + /// of `bytes`, and the remainder returned alongside `Self`. + fn from_bytes_copy_prefix(bytes: &[u8]) -> Option<(Self, &[u8])> + where + Self: Sized, + { + let (prefix, remainder) = bytes.split_at_checked(size_of::())?; + + Self::from_bytes_copy(prefix).map(|s| (s, remainder)) + } +} + +macro_rules! impl_frombytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? FromBytes for $t {})* + }; +} + +impl_frombytes! { + // SAFETY: Inhabited ZSTs only have one possible bit pattern, and these two have no invariant. + (), + {} core::marker::PhantomData, + + // SAFETY: All bit patterns are acceptable values of the types below. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + + // SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit + // patterns are also acceptable for arrays of that type. + {} [T], + {} [T; N], +} + +/// Types that can be viewed as an immutable slice of initialized bytes. +/// +/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This +/// means that it should not have any padding, as padding bytes are uninitialized. Reading +/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive +/// information on the stack to userspace. +/// +/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered +/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so +/// this is a correctness requirement, but not a safety requirement. +/// +/// # Safety +/// +/// Values of this type may not contain any uninitialized bytes. This type must not have interior +/// mutability. +pub unsafe trait AsBytes { + /// Returns `self` as a slice of bytes. + fn as_bytes(&self) -> &[u8] { + // CAST: `Self` implements `AsBytes` thus all bytes of `self` are initialized. + let data = core::ptr::from_ref(self).cast::(); + let len = core::mem::size_of_val(self); + + // SAFETY: `data` is non-null and valid for reads of `len * sizeof::()` bytes. + unsafe { core::slice::from_raw_parts(data, len) } + } + + /// Returns `self` as a mutable slice of bytes. + fn as_bytes_mut(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + // CAST: `Self` implements both `AsBytes` and `FromBytes` thus making `Self` + // bi-directionally transmutable to `[u8; size_of_val(self)]`. + let data = core::ptr::from_mut(self).cast::(); + let len = core::mem::size_of_val(self); + + // SAFETY: `data` is non-null and valid for read and writes of `len * sizeof::()` + // bytes. + unsafe { core::slice::from_raw_parts_mut(data, len) } + } +} + +macro_rules! impl_asbytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? AsBytes for $t {})* + }; +} + +impl_asbytes! { + // SAFETY: Inhabited ZSTs only have one possible bit pattern, and these two have no invariant. + (), + {} core::marker::PhantomData, + + // SAFETY: Instances of the following types have no uninitialized portions. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + bool, + char, + str, + + // SAFETY: If individual values in an array have no uninitialized portions, then the array + // itself does not have any uninitialized portions either. + {} [T], + {} [T; N], +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs new file mode 100644 index 000000000..132dd428c --- /dev/null +++ b/rust/kernel/types.rs @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel types. + +use crate::ffi::c_void; +use core::{ + cell::UnsafeCell, + marker::{PhantomData, PhantomPinned}, + mem::MaybeUninit, + ops::{Deref, DerefMut}, +}; +use pin_init::{PinInit, Wrapper, Zeroable}; + +#[doc(hidden)] +pub mod for_lt; +pub use for_lt::{ + CovariantForLt, + ForLt, // +}; + +/// Used to transfer ownership to and from foreign (non-Rust) languages. +/// +/// Ownership is transferred from Rust to a foreign language by calling [`Self::into_foreign`] and +/// later may be transferred back to Rust by calling [`Self::from_foreign`]. +/// +/// This trait is meant to be used in cases when Rust objects are stored in C objects and +/// eventually "freed" back to Rust. +/// +/// # Safety +/// +/// - Implementations must satisfy the guarantees of [`Self::into_foreign`]. +pub unsafe trait ForeignOwnable: Sized { + /// The alignment of pointers returned by `into_foreign`. + const FOREIGN_ALIGN: usize; + + /// Type used to immutably borrow a value that is currently foreign-owned. + type Borrowed<'a> + where + Self: 'a; + + /// Type used to mutably borrow a value that is currently foreign-owned. + type BorrowedMut<'a> + where + Self: 'a; + + /// Converts a Rust-owned object to a foreign-owned one. + /// + /// The foreign representation is a pointer to void. Aside from the guarantees listed below, + /// there are no other guarantees for this pointer. For example, it might be invalid, dangling + /// or pointing to uninitialized memory. Using it in any way except for [`from_foreign`], + /// [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can result in undefined behavior. + /// + /// # Guarantees + /// + /// - Minimum alignment of returned pointer is [`Self::FOREIGN_ALIGN`]. + /// - The returned pointer is not null. + /// + /// [`from_foreign`]: Self::from_foreign + /// [`try_from_foreign`]: Self::try_from_foreign + /// [`borrow`]: Self::borrow + /// [`borrow_mut`]: Self::borrow_mut + fn into_foreign(self) -> *mut c_void; + + /// Converts a foreign-owned object back to a Rust-owned one. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and it + /// must not be passed to `from_foreign` more than once. + /// + /// [`into_foreign`]: Self::into_foreign + unsafe fn from_foreign(ptr: *mut c_void) -> Self; + + /// Tries to convert a foreign-owned object back to a Rust-owned one. + /// + /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr` + /// is null. + /// + /// # Safety + /// + /// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`]. + /// + /// [`from_foreign`]: Self::from_foreign + unsafe fn try_from_foreign(ptr: *mut c_void) -> Option { + if ptr.is_null() { + None + } else { + // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements + // of `from_foreign` given the safety requirements of this function. + unsafe { Some(Self::from_foreign(ptr)) } + } + } + + /// Borrows a foreign-owned object immutably. + /// + /// This method provides a way to access a foreign-owned value from Rust immutably. It provides + /// you with exactly the same abilities as an `&Self` when the value is Rust-owned. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>; + + /// Borrows a foreign-owned object mutably. + /// + /// This method provides a way to access a foreign-owned value from Rust mutably. It provides + /// you with exactly the same abilities as an `&mut Self` when the value is Rust-owned, except + /// that the address of the object must not be changed. + /// + /// Note that for types like [`Arc`], an `&mut Arc` only gives you immutable access to the + /// inner value, so this method also only provides immutable access in that case. + /// + /// In the case of `Box`, this method gives you the ability to modify the inner `T`, but it + /// does not let you change the box itself. That is, you cannot change which allocation the box + /// points at. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// The lifetime `'a` must not overlap with the lifetime of any other call to [`borrow`] or + /// `borrow_mut` on the same object. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + /// [`borrow`]: Self::borrow + /// [`Arc`]: crate::sync::Arc + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>; +} + +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `()`. +unsafe impl ForeignOwnable for () { + const FOREIGN_ALIGN: usize = core::mem::align_of::<()>(); + type Borrowed<'a> = (); + type BorrowedMut<'a> = (); + + fn into_foreign(self) -> *mut c_void { + core::ptr::NonNull::dangling().as_ptr() + } + + unsafe fn from_foreign(_: *mut c_void) -> Self {} + + unsafe fn borrow<'a>(_: *mut c_void) -> Self::Borrowed<'a> {} + unsafe fn borrow_mut<'a>(_: *mut c_void) -> Self::BorrowedMut<'a> {} +} + +/// Runs a cleanup function/closure when dropped. +/// +/// The [`ScopeGuard::dismiss`] function prevents the cleanup function from running. +/// +/// # Examples +/// +/// In the example below, we have multiple exit paths and we want to log regardless of which one is +/// taken: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example1(arg: bool) { +/// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n")); +/// +/// if arg { +/// return; +/// } +/// +/// pr_info!("Do something...\n"); +/// } +/// +/// # example1(false); +/// # example1(true); +/// ``` +/// +/// In the example below, we want to log the same message on all early exits but a different one on +/// the main exit path: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example2(arg: bool) { +/// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n")); +/// +/// if arg { +/// return; +/// } +/// +/// // (Other early returns...) +/// +/// log.dismiss(); +/// pr_info!("example2 no early return\n"); +/// } +/// +/// # example2(false); +/// # example2(true); +/// ``` +/// +/// In the example below, we need a mutable object (the vector) to be accessible within the log +/// function, so we wrap it in the [`ScopeGuard`]: +/// +/// ``` +/// # use kernel::types::ScopeGuard; +/// fn example3(arg: bool) -> Result { +/// let mut vec = +/// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); +/// +/// vec.push(10u8, GFP_KERNEL)?; +/// if arg { +/// return Ok(()); +/// } +/// vec.push(20u8, GFP_KERNEL)?; +/// Ok(()) +/// } +/// +/// # assert_eq!(example3(false), Ok(())); +/// # assert_eq!(example3(true), Ok(())); +/// ``` +/// +/// # Invariants +/// +/// The value stored in the struct is nearly always `Some(_)`, except between +/// [`ScopeGuard::dismiss`] and [`ScopeGuard::drop`]: in this case, it will be `None` as the value +/// will have been returned to the caller. Since [`ScopeGuard::dismiss`] consumes the guard, +/// callers won't be able to use it anymore. +pub struct ScopeGuard(Option<(T, F)>); + +impl ScopeGuard { + /// Creates a new guarded object wrapping the given data and with the given cleanup function. + pub fn new_with_data(data: T, cleanup_func: F) -> Self { + // INVARIANT: The struct is being initialised with `Some(_)`. + Self(Some((data, cleanup_func))) + } + + /// Prevents the cleanup function from running and returns the guarded data. + pub fn dismiss(mut self) -> T { + // INVARIANT: This is the exception case in the invariant; it is not visible to callers + // because this function consumes `self`. + self.0.take().unwrap().0 + } +} + +impl ScopeGuard<(), fn(())> { + /// Creates a new guarded object with the given cleanup function. + pub fn new(cleanup: impl FnOnce()) -> ScopeGuard<(), impl FnOnce(())> { + ScopeGuard::new_with_data((), move |()| cleanup()) + } +} + +impl Deref for ScopeGuard { + type Target = T; + + fn deref(&self) -> &T { + // The type invariants guarantee that `unwrap` will succeed. + &self.0.as_ref().unwrap().0 + } +} + +impl DerefMut for ScopeGuard { + fn deref_mut(&mut self) -> &mut T { + // The type invariants guarantee that `unwrap` will succeed. + &mut self.0.as_mut().unwrap().0 + } +} + +impl Drop for ScopeGuard { + fn drop(&mut self) { + // Run the cleanup function if one is still present. + if let Some((data, cleanup)) = self.0.take() { + cleanup(data) + } + } +} + +/// Stores an opaque value. +/// +/// [`Opaque`] is meant to be used with FFI objects that are never interpreted by Rust code. +/// +/// It is used to wrap structs from the C side, like for example `Opaque`. +/// It gets rid of all the usual assumptions that Rust has for a value: +/// +/// * The value is allowed to be uninitialized (for example have invalid bit patterns: `3` for a +/// [`bool`]). +/// * The value is allowed to be mutated, when a `&Opaque` exists on the Rust side. +/// * No uniqueness for mutable references: it is fine to have multiple `&mut Opaque` point to +/// the same value. +/// * The value is not allowed to be shared with other threads (i.e. it is `!Sync`). +/// +/// This has to be used for all values that the C side has access to, because it can't be ensured +/// that the C side is adhering to the usual constraints that Rust needs. +/// +/// Using [`Opaque`] allows to continue to use references on the Rust side even for values shared +/// with C. +/// +/// # Examples +/// +/// ``` +/// use kernel::types::Opaque; +/// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side +/// # // knows. +/// # mod bindings { +/// # pub struct Foo { +/// # pub val: u8, +/// # } +/// # } +/// +/// // `foo.val` is assumed to be handled on the C side, so we use `Opaque` to wrap it. +/// pub struct Foo { +/// foo: Opaque, +/// } +/// +/// impl Foo { +/// pub fn get_val(&self) -> u8 { +/// let ptr = Opaque::get(&self.foo); +/// +/// // SAFETY: `Self` is valid from C side. +/// unsafe { (*ptr).val } +/// } +/// } +/// +/// // Create an instance of `Foo` with the `Opaque` wrapper. +/// let foo = Foo { +/// foo: Opaque::new(bindings::Foo { val: 0xdb }), +/// }; +/// +/// assert_eq!(foo.get_val(), 0xdb); +/// ``` +#[repr(transparent)] +pub struct Opaque { + value: UnsafeCell>, + _pin: PhantomPinned, +} + +// SAFETY: `Opaque` allows the inner value to be any bit pattern, including all zeros. +unsafe impl Zeroable for Opaque {} + +impl Opaque { + /// Creates a new opaque value. + pub const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::new(value)), + _pin: PhantomPinned, + } + } + + /// Creates an uninitialised value. + pub const fn uninit() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + _pin: PhantomPinned, + } + } + + /// Creates a new zeroed opaque value. + pub const fn zeroed() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::zeroed()), + _pin: PhantomPinned, + } + } + + /// Creates a pin-initializer from the given initializer closure. + /// + /// The returned initializer calls the given closure with the pointer to the inner `T` of this + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. + /// + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs + /// to verify at that point that the inner value is valid. + pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit { + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully + // initialize the `T`. + unsafe { + pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { + init_func(Self::cast_into(slot)); + Ok(()) + }) + } + } + + /// Creates a fallible pin-initializer from the given initializer closure. + /// + /// The returned initializer calls the given closure with the pointer to the inner `T` of this + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. + /// + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs + /// to verify at that point that the inner value is valid. + pub fn try_ffi_init( + init_func: impl FnOnce(*mut T) -> Result<(), E>, + ) -> impl PinInit { + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully + // initialize the `T`. + unsafe { + pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::cast_into(slot))) + } + } + + /// Returns a raw pointer to the opaque data. + pub const fn get(&self) -> *mut T { + UnsafeCell::get(&self.value).cast::() + } + + /// Gets the value behind `this`. + /// + /// This function is useful to get access to the value without creating intermediate + /// references. + pub const fn cast_into(this: *const Self) -> *mut T { + UnsafeCell::raw_get(this.cast::>>()).cast::() + } + + /// The opposite operation of [`Opaque::cast_into`]. + pub const fn cast_from(this: *const T) -> *const Self { + this.cast() + } +} + +impl Wrapper for Opaque { + /// Create an opaque pin-initializer from the given pin-initializer. + fn pin_init(init: impl PinInit) -> impl PinInit { + Self::try_ffi_init(|slot: *mut T| { + // SAFETY: + // - `slot` is a valid pointer to uninitialized memory, + // - `slot` is not accessed on error, + // - `slot` is pinned in memory. + unsafe { pin_init::raw_try_init(slot, init) } + }) + } +} + +/// Zero-sized type to mark types not [`Send`]. +/// +/// Add this type as a field to your struct if your type should not be sent to a different task. +/// Since [`Send`] is an auto trait, adding a single field that is `!Send` will ensure that the +/// whole type is `!Send`. +/// +/// If a type is `!Send` it is impossible to give control over an instance of the type to another +/// task. This is useful to include in types that store or reference task-local information. A file +/// descriptor is an example of such task-local information. +/// +/// This type also makes the type `!Sync`, which prevents immutable access to the value from +/// several threads in parallel. +pub type NotThreadSafe = PhantomData<*mut ()>; + +/// Used to construct instances of type [`NotThreadSafe`] similar to how `PhantomData` is +/// constructed. +/// +/// [`NotThreadSafe`]: type@NotThreadSafe +#[allow(non_upper_case_globals)] +pub const NotThreadSafe: NotThreadSafe = PhantomData; diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs new file mode 100644 index 000000000..b8f422c80 --- /dev/null +++ b/rust/kernel/types/for_lt.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and +//! macros. +//! +//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) / +//! [`CovariantForLt!`](macro@CovariantForLt) directly. + +use core::marker::PhantomData; + +/// Representation of types generic over a lifetime. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is +/// provided to obtain a type that implements this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that ` TypeThatUse<'a>) as ForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(TypeThatUse<'_>) // Equivalent to `ForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(&u32) // Equivalent to `ForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. +/// # >(); +/// ``` +pub trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; +} +pub use macros::ForLt; + +/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter. +/// +/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be +/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. +/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements +/// this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that +/// ` TypeThatUse<'a>) as CovariantForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`. +/// # >(); +/// ``` +/// +/// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. +/// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove +/// it. +/// +/// ```ignore,compile_fail +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # expect_lt::< +/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// # >(); +/// ``` +/// +/// There is a limitation if the type refers to generic parameters; if the macro cannot prove the +/// covariance syntactically, the emitted checks will fail the compilation as it needs to refer to +/// the generic parameter but is in a separate item. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// fn expect_lt() {} +/// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] +/// fn generic_fn() { +/// // Syntactically proven by the macro +/// expect_lt::(); +/// // Syntactically proven by the macro +/// expect_lt::)>(); +/// // Cannot be syntactically proven, need to check covariance of `KBox` +/// // expect_lt::)>(); +/// } +/// ``` +/// +/// # Safety +/// +/// `Self::Of<'a>` must be covariant over the lifetime `'a`. +pub unsafe trait CovariantForLt: ForLt { + /// Cast a reference to a shorter lifetime. + #[inline(always)] + fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { + // SAFETY: This is sound as this trait guarantees covariance. + unsafe { core::mem::transmute(long) } + } +} +pub use macros::CovariantForLt; + +/// This is intended to be an "unsafe-to-refer-to" type. +/// +/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt) +/// macros. +/// +/// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. +/// +/// `WF` is a type that the macro can use to assert some specific type is well-formed. +/// +/// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove +/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`; +/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof. +#[doc(hidden)] +pub struct UnsafeForLtImpl(PhantomData<(WF, T)>); + +// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB. +#[doc(hidden)] +pub trait WithLt<'a> { + type Of: 'a; +} + +impl WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl { + type Of<'a> = >::Of; +} + +// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic +// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl. +unsafe impl WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl {} diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs new file mode 100644 index 000000000..5f6c4d7a1 --- /dev/null +++ b/rust/kernel/uaccess.rs @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Slices to user space memory regions. +//! +//! C header: [`include/linux/uaccess.h`](srctree/include/linux/uaccess.h) + +use crate::{ + alloc::{Allocator, Flags}, + bindings, + dma::Coherent, + error::Result, + ffi::{c_char, c_void}, + fs::file, + prelude::*, + ptr::KnownSize, + transmute::{AsBytes, FromBytes}, +}; +use core::mem::{size_of, MaybeUninit}; + +/// A pointer into userspace. +/// +/// This is the Rust equivalent to C pointers tagged with `__user`. +#[repr(transparent)] +#[derive(Copy, Clone, Zeroable)] +pub struct UserPtr(*mut c_void); + +impl UserPtr { + /// Create a `UserPtr` from an integer representing the userspace address. + #[inline] + pub fn from_addr(addr: usize) -> Self { + Self(addr as *mut c_void) + } + + /// Create a `UserPtr` from a pointer representing the userspace address. + #[inline] + pub fn from_ptr(addr: *mut c_void) -> Self { + Self(addr) + } + + /// Cast this userspace pointer to a raw const void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_const_ptr(self) -> *const c_void { + self.0 + } + + /// Cast this userspace pointer to a raw mutable void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_mut_ptr(self) -> *mut c_void { + self.0 + } + + /// Increment this user pointer by `add` bytes. + /// + /// This addition is wrapping, so wrapping around the address space does not result in a panic + /// even if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled. + #[inline] + pub fn wrapping_byte_add(self, add: usize) -> UserPtr { + UserPtr(self.0.wrapping_byte_add(add)) + } +} + +/// A pointer to an area in userspace memory, which can be either read-only or read-write. +/// +/// All methods on this struct are safe: attempting to read or write on bad addresses (either out of +/// the bound of the slice or unmapped addresses) will return [`EFAULT`]. Concurrent access, +/// *including data races to/from userspace memory*, is permitted, because fundamentally another +/// userspace thread/process could always be modifying memory at the same time (in the same way that +/// userspace Rust's [`std::io`] permits data races with the contents of files on disk). In the +/// presence of a race, the exact byte values read/written are unspecified but the operation is +/// well-defined. Kernelspace code should validate its copy of data after completing a read, and not +/// expect that multiple reads of the same address will return the same value. +/// +/// These APIs are designed to make it difficult to accidentally write TOCTOU (time-of-check to +/// time-of-use) bugs. Every time a memory location is read, the reader's position is advanced by +/// the read length and the next read will start from there. This helps prevent accidentally reading +/// the same location twice and causing a TOCTOU bug. +/// +/// Creating a [`UserSliceReader`] and/or [`UserSliceWriter`] consumes the `UserSlice`, helping +/// ensure that there aren't multiple readers or writers to the same location. +/// +/// If double-fetching a memory location is necessary for some reason, then that is done by creating +/// multiple readers to the same memory location, e.g. using [`clone_reader`]. +/// +/// # Examples +/// +/// Takes a region of userspace memory from the current process, and modify it by adding one to +/// every byte in the region. +/// +/// ```no_run +/// use kernel::ffi::c_void; +/// use kernel::uaccess::{UserPtr, UserSlice}; +/// +/// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result { +/// let (read, mut write) = UserSlice::new(uptr, len).reader_writer(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// for b in &mut buf { +/// *b = b.wrapping_add(1); +/// } +/// +/// write.write_slice(&buf)?; +/// Ok(()) +/// } +/// ``` +/// +/// Example illustrating a TOCTOU (time-of-check to time-of-use) bug. +/// +/// ```no_run +/// use kernel::ffi::c_void; +/// use kernel::uaccess::{UserPtr, UserSlice}; +/// +/// /// Returns whether the data in this region is valid. +/// fn is_valid(uptr: UserPtr, len: usize) -> Result { +/// let read = UserSlice::new(uptr, len).reader(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// todo!() +/// } +/// +/// /// Returns the bytes behind this user pointer if they are valid. +/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { +/// if !is_valid(uptr, len)? { +/// return Err(EINVAL); +/// } +/// +/// let read = UserSlice::new(uptr, len).reader(); +/// +/// let mut buf = KVec::new(); +/// read.read_all(&mut buf, GFP_KERNEL)?; +/// +/// // THIS IS A BUG! The bytes could have changed since we checked them. +/// // +/// // To avoid this kind of bug, don't call `UserSlice::new` multiple +/// // times with the same address. +/// Ok(buf) +/// } +/// ``` +/// +/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html +/// [`clone_reader`]: UserSliceReader::clone_reader +pub struct UserSlice { + ptr: UserPtr, + length: usize, +} + +impl UserSlice { + /// Constructs a user slice from a raw pointer and a length in bytes. + /// + /// Constructing a [`UserSlice`] performs no checks on the provided address and length, it can + /// safely be constructed inside a kernel thread with no current userspace process. Reads and + /// writes wrap the kernel APIs `copy_from_user` and `copy_to_user`, which check the memory map + /// of the current process and enforce that the address range is within the user range (no + /// additional calls to `access_ok` are needed). Validity of the pointer is checked when you + /// attempt to read or write, not in the call to `UserSlice::new`. + /// + /// Callers must be careful to avoid time-of-check-time-of-use (TOCTOU) issues. The simplest way + /// is to create a single instance of [`UserSlice`] per user memory block as it reads each byte + /// at most once. + pub fn new(ptr: UserPtr, length: usize) -> Self { + UserSlice { ptr, length } + } + + /// Reads the entirety of the user slice, appending it to the end of the provided buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address. + pub fn read_all(self, buf: &mut Vec, flags: Flags) -> Result { + self.reader().read_all(buf, flags) + } + + /// Constructs a [`UserSliceReader`]. + pub fn reader(self) -> UserSliceReader { + UserSliceReader { + ptr: self.ptr, + length: self.length, + } + } + + /// Constructs a [`UserSliceWriter`]. + pub fn writer(self) -> UserSliceWriter { + UserSliceWriter { + ptr: self.ptr, + length: self.length, + } + } + + /// Constructs both a [`UserSliceReader`] and a [`UserSliceWriter`]. + /// + /// Usually when this is used, you will first read the data, and then overwrite it afterwards. + pub fn reader_writer(self) -> (UserSliceReader, UserSliceWriter) { + ( + UserSliceReader { + ptr: self.ptr, + length: self.length, + }, + UserSliceWriter { + ptr: self.ptr, + length: self.length, + }, + ) + } +} + +/// A reader for [`UserSlice`]. +/// +/// Used to incrementally read from the user slice. +pub struct UserSliceReader { + ptr: UserPtr, + length: usize, +} + +impl UserSliceReader { + /// Skip the provided number of bytes. + /// + /// Returns an error if skipping more than the length of the buffer. + pub fn skip(&mut self, num_skip: usize) -> Result { + // Update `self.length` first since that's the fallible part of this operation. + self.length = self.length.checked_sub(num_skip).ok_or(EFAULT)?; + self.ptr = self.ptr.wrapping_byte_add(num_skip); + Ok(()) + } + + /// Create a reader that can access the same range of data. + /// + /// Reading from the clone does not advance the current reader. + /// + /// The caller should take care to not introduce TOCTOU issues, as described in the + /// documentation for [`UserSlice`]. + pub fn clone_reader(&self) -> UserSliceReader { + UserSliceReader { + ptr: self.ptr, + length: self.length, + } + } + + /// Returns the number of bytes left to be read from this reader. + /// + /// Note that even reading less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.length + } + + /// Returns `true` if no data is available in the io buffer. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Reads raw data from the user slice into a kernel buffer. + /// + /// For a version that uses `&mut [u8]`, please see [`UserSliceReader::read_slice`]. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. + /// + /// # Guarantees + /// + /// After a successful call to this method, all bytes in `out` are initialized. + pub fn read_raw(&mut self, out: &mut [MaybeUninit]) -> Result { + let len = out.len(); + let out_ptr = out.as_mut_ptr().cast::(); + if len > self.length { + return Err(EFAULT); + } + // SAFETY: `out_ptr` points into a mutable slice of length `len`, so we may write + // that many bytes to it. + let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr.as_const_ptr(), len) }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } + + /// Reads raw data from the user slice into a kernel buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. + pub fn read_slice(&mut self, out: &mut [u8]) -> Result { + // SAFETY: The types are compatible and `read_raw` doesn't write uninitialized bytes to + // `out`. + let out = unsafe { &mut *(core::ptr::from_mut(out) as *mut [MaybeUninit]) }; + self.read_raw(out) + } + + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice`] but considers the given `offset` into `out` and + /// truncates the read to the boundaries of `self` and `out`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_partial(&mut self, out: &mut [u8], offset: usize) -> Result { + let end = offset.saturating_add(self.len()).min(out.len()); + + let Some(dst) = out.get_mut(offset..end) else { + return Ok(0); + }; + + self.read_slice(dst)?; + Ok(dst.len()) + } + + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes read. + /// + /// This is equivalent to C's `simple_write_to_buffer()`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_file(&mut self, out: &mut [u8], offset: &mut file::Offset) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let read = self.read_slice_partial(out, offset_index)?; + + // OVERFLOW: `offset + read <= data.len() <= isize::MAX <= Offset::MAX` + *offset += read as i64; + + Ok(read) + } + + /// Reads a value of the specified type. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of + /// bounds of this [`UserSliceReader`]. + pub fn read(&mut self) -> Result { + let len = size_of::(); + if len > self.length { + return Err(EFAULT); + } + let mut out: MaybeUninit = MaybeUninit::uninit(); + // SAFETY: The local variable `out` is valid for writing `size_of::()` bytes. + // + // By using the _copy_from_user variant, we skip the check_object_size check that verifies + // the kernel pointer. This mirrors the logic on the C side that skips the check when the + // length is a compile-time constant. + let res = unsafe { + bindings::_copy_from_user( + out.as_mut_ptr().cast::(), + self.ptr.as_const_ptr(), + len, + ) + }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + // SAFETY: The read above has initialized all bytes in `out`, and since `T` implements + // `FromBytes`, any bit-pattern is a valid value for this type. + Ok(unsafe { out.assume_init() }) + } + + /// Reads the entirety of the user slice, appending it to the end of the provided buffer. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address. + pub fn read_all(mut self, buf: &mut Vec, flags: Flags) -> Result { + let len = self.length; + buf.reserve(len, flags)?; + + // The call to `reserve` was successful, so the spare capacity is at least `len` bytes long. + self.read_raw(&mut buf.spare_capacity_mut()[..len])?; + + // SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the + // vector have been initialized. + unsafe { buf.inc_len(len) }; + Ok(()) + } + + /// Read a NUL-terminated string from userspace and return it. + /// + /// The string is read into `buf` and a NUL-terminator is added if the end of `buf` is reached. + /// Since there must be space to add a NUL-terminator, the buffer must not be empty. The + /// returned `&CStr` points into `buf`. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address (some data may have been + /// copied). + #[doc(alias = "strncpy_from_user")] + pub fn strcpy_into_buf<'buf>(self, buf: &'buf mut [u8]) -> Result<&'buf CStr> { + if buf.is_empty() { + return Err(EINVAL); + } + + // SAFETY: The types are compatible and `strncpy_from_user` doesn't write uninitialized + // bytes to `buf`. + let mut dst = unsafe { &mut *(core::ptr::from_mut(buf) as *mut [MaybeUninit]) }; + + // We never read more than `self.length` bytes. + if dst.len() > self.length { + dst = &mut dst[..self.length]; + } + + let mut len = raw_strncpy_from_user(dst, self.ptr)?; + if len < dst.len() { + // Add one to include the NUL-terminator. + len += 1; + } else if len < buf.len() { + // This implies that `len == dst.len() < buf.len()`. + // + // This means that we could not fill the entire buffer, but we had to stop reading + // because we hit the `self.length` limit of this `UserSliceReader`. Since we did not + // fill the buffer, we treat this case as if we tried to read past the `self.length` + // limit and received a page fault, which is consistent with other `UserSliceReader` + // methods that also return page faults when you exceed `self.length`. + return Err(EFAULT); + } else { + // This implies that `len == buf.len()`. + // + // This means that we filled the buffer exactly. In this case, we add a NUL-terminator + // and return it. Unlike the `len < dst.len()` branch, don't modify `len` because it + // already represents the length including the NUL-terminator. + // + // SAFETY: Due to the check at the beginning, the buffer is not empty. + unsafe { *buf.last_mut().unwrap_unchecked() = 0 }; + } + + // This method consumes `self`, so it can only be called once, thus we do not need to + // update `self.length`. This sidesteps concerns such as whether `self.length` should be + // incremented by `len` or `len-1` in the `len == buf.len()` case. + + // SAFETY: There are two cases: + // * If we hit the `len < dst.len()` case, then `raw_strncpy_from_user` guarantees that + // this slice contains exactly one NUL byte at the end of the string. + // * Otherwise, `raw_strncpy_from_user` guarantees that the string contained no NUL bytes, + // and we have since added a NUL byte at the end. + Ok(unsafe { CStr::from_bytes_with_nul_unchecked(&buf[..len]) }) + } +} + +/// A writer for [`UserSlice`]. +/// +/// Used to incrementally write into the user slice. +pub struct UserSliceWriter { + ptr: UserPtr, + length: usize, +} + +impl UserSliceWriter { + /// Returns the amount of space remaining in this buffer. + /// + /// Note that even writing less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.length + } + + /// Returns `true` if no more data can be written to this buffer. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Low-level write from a raw pointer. + /// + /// # Safety + /// + /// The caller must ensure that `from` is valid for reads of `len` bytes. + unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { + if len > self.length { + return Err(EFAULT); + } + + // SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's + // safety contract). + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } + + /// Writes raw data to this user pointer from a kernel buffer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for + // reading `data.len()` bytes. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to this user pointer from a DMA coherent allocation. + /// + /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. + /// + /// # Errors + /// + /// - [`EOVERFLOW`]: `offset + count` overflows. + /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the + /// size of the user-space buffer. + /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this + /// [`UserSliceWriter`]. + /// + /// This call may modify the associated userspace slice even if it returns an error. + /// + /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, + /// the copied data may be inconsistent, but this does not cause undefined behavior. + /// + /// # Example + /// + /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: + /// + /// ```no_run + /// use kernel::uaccess::UserSliceWriter; + /// use kernel::dma::Coherent; + /// + /// fn copy_dma_to_user( + /// mut writer: UserSliceWriter, + /// alloc: &Coherent<[u8]>, + /// ) -> Result { + /// writer.write_dma(alloc, 0, 256) + /// } + /// ``` + pub fn write_dma( + &mut self, + alloc: &Coherent, + offset: usize, + count: usize, + ) -> Result { + let len = alloc.size(); + if offset.checked_add(count).ok_or(EOVERFLOW)? > len { + return Err(ERANGE); + } + + if count > self.len() { + return Err(ERANGE); + } + + // SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as + // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. + let src_ptr = unsafe { alloc.as_ptr().cast::().add(offset) }; + + // Note: Use `write_raw` instead of `write_slice` because the allocation is coherent + // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over + // such volatile memory. + // + // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). + unsafe { self.write_raw(src_ptr, count) } + } + + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and + /// truncates the write to the boundaries of `self` and `data`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result { + let end = offset.saturating_add(self.len()).min(data.len()); + + let Some(src) = data.get(offset..end) else { + return Ok(0); + }; + + self.write_slice(src)?; + Ok(src.len()) + } + + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes written. + /// + /// This is equivalent to C's `simple_read_from_buffer()`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_file(&mut self, data: &[u8], offset: &mut file::Offset) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let written = self.write_slice_partial(data, offset_index)?; + + // OVERFLOW: `offset + written <= data.len() <= isize::MAX <= Offset::MAX` + *offset += written as i64; + + Ok(written) + } + + /// Writes the provided Rust value to this userspace pointer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write(&mut self, value: &T) -> Result { + let len = size_of::(); + if len > self.length { + return Err(EFAULT); + } + // SAFETY: The reference points to a value of type `T`, so it is valid for reading + // `size_of::()` bytes. + // + // By using the _copy_to_user variant, we skip the check_object_size check that verifies the + // kernel pointer. This mirrors the logic on the C side that skips the check when the length + // is a compile-time constant. + let res = unsafe { + bindings::_copy_to_user( + self.ptr.as_mut_ptr(), + core::ptr::from_ref(value).cast::(), + len, + ) + }; + if res != 0 { + return Err(EFAULT); + } + self.ptr = self.ptr.wrapping_byte_add(len); + self.length -= len; + Ok(()) + } +} + +/// Reads a nul-terminated string into `dst` and returns the length. +/// +/// This reads from userspace until a NUL byte is encountered, or until `dst.len()` bytes have been +/// read. Fails with [`EFAULT`] if a read happens on a bad address (some data may have been +/// copied). When the end of the buffer is encountered, no NUL byte is added, so the string is +/// *not* guaranteed to be NUL-terminated when `Ok(dst.len())` is returned. +/// +/// # Guarantees +/// +/// When this function returns `Ok(len)`, it is guaranteed that the first `len` bytes of `dst` are +/// initialized and non-zero. Furthermore, if `len < dst.len()`, then `dst[len]` is a NUL byte. +#[inline] +fn raw_strncpy_from_user(dst: &mut [MaybeUninit], src: UserPtr) -> Result { + // CAST: Slice lengths are guaranteed to be `<= isize::MAX`. + let len = dst.len() as isize; + + // SAFETY: `dst` is valid for writing `dst.len()` bytes. + let res = unsafe { + bindings::strncpy_from_user( + dst.as_mut_ptr().cast::(), + src.as_const_ptr().cast::(), + len, + ) + }; + + if res < 0 { + return Err(Error::from_errno(res as i32)); + } + + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res <= len); + + // GUARANTEES: `strncpy_from_user` was successful, so `dst` has contents in accordance with the + // guarantees of this function. + Ok(res as usize) +} diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs new file mode 100644 index 000000000..993760ff3 --- /dev/null +++ b/rust/kernel/usb.rs @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (C) 2025 Collabora Ltd. + +//! Abstractions for the USB bus. +//! +//! C header: [`include/linux/usb.h`](srctree/include/linux/usb.h) + +use crate::{ + bindings, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + driver, + error::{ + from_result, + to_result, // + }, + prelude::*, + sync::aref::AlwaysRefCounted, + types::Opaque, + ThisModule, // +}; +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::NonNull, // +}; + +/// An adapter for the registration of USB drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::usb_driver` is a C type declared as `repr(C)`. +// - `T::Data` is the type of the driver's device private data. +// - `struct usb_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::usb_driver; + type DriverData<'bound> = T::Data<'bound>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + udrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct usb_driver` on initialization. + unsafe { + (*udrv.get()).name = name.as_char_ptr(); + (*udrv.get()).probe = Some(Self::probe_callback); + (*udrv.get()).disconnect = Some(Self::disconnect_callback); + (*udrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { + bindings::usb_register_driver(udrv.get(), module.as_ptr(), name.as_char_ptr()) + }) + } + + unsafe fn unregister(udrv: &Opaque) { + // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::usb_deregister(udrv.get()) }; + } +} + +impl Adapter { + extern "C" fn probe_callback( + intf: *mut bindings::usb_interface, + id: *const bindings::usb_device_id, + ) -> kernel::ffi::c_int { + // SAFETY: The USB core only ever calls the probe callback with a valid pointer to a + // `struct usb_interface` and `struct usb_device_id`. + // + // INVARIANT: `intf` is valid for the duration of `probe_callback()`. + let intf = unsafe { &*intf.cast::>>() }; + + from_result(|| { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct usb_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::() }; + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. It + // can also come from dynamic IDs, which will ensure that `driver_data` exists in + // `T::ID_TABLE` or is 0. + let info = unsafe { id.info_unchecked_opt::() }; + let data = T::probe(intf, id, info); + + let dev: &device::Device> = intf.as_ref(); + dev.set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) { + // SAFETY: The USB core only ever calls the disconnect callback with a valid pointer to a + // `struct usb_interface`. + // + // INVARIANT: `intf` is valid for the duration of `disconnect_callback()`. + let intf = unsafe { &*intf.cast::>>() }; + + let dev: &device::Device> = intf.as_ref(); + + // SAFETY: `disconnect_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let data = unsafe { dev.drvdata_borrow::>() }; + + T::disconnect(intf, data); + } +} + +/// Abstraction for the USB device ID structure, i.e. [`struct usb_device_id`]. +/// +/// [`struct usb_device_id`]: https://docs.kernel.org/driver-api/basics.html#c.usb_device_id +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::usb_device_id); + +impl DeviceId { + /// Equivalent to C's `USB_DEVICE` macro. + pub const fn from_id(vendor: u16, product: u16) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16, + idVendor: vendor, + idProduct: product, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_VER` macro. + pub const fn from_device_ver(vendor: u16, product: u16, bcd_lo: u16, bcd_hi: u16) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION as u16, + idVendor: vendor, + idProduct: product, + bcdDevice_lo: bcd_lo, + bcdDevice_hi: bcd_hi, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INFO` macro. + pub const fn from_device_info(class: u8, subclass: u8, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_DEV_INFO as u16, + bDeviceClass: class, + bDeviceSubClass: subclass, + bDeviceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_INTERFACE_INFO` macro. + pub const fn from_interface_info(class: u8, subclass: u8, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: bindings::USB_DEVICE_ID_MATCH_INT_INFO as u16, + bInterfaceClass: class, + bInterfaceSubClass: subclass, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_CLASS` macro. + pub const fn from_device_interface_class(vendor: u16, product: u16, class: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_CLASS) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceClass: class, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_PROTOCOL` macro. + pub const fn from_device_interface_protocol(vendor: u16, product: u16, protocol: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_PROTOCOL) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_INTERFACE_NUMBER` macro. + pub const fn from_device_interface_number(vendor: u16, product: u16, number: u8) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_DEVICE + | bindings::USB_DEVICE_ID_MATCH_INT_NUMBER) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceNumber: number, + ..pin_init::zeroed() + }) + } + + /// Equivalent to C's `USB_DEVICE_AND_INTERFACE_INFO` macro. + pub const fn from_device_and_interface_info( + vendor: u16, + product: u16, + class: u8, + subclass: u8, + protocol: u8, + ) -> Self { + Self(bindings::usb_device_id { + match_flags: (bindings::USB_DEVICE_ID_MATCH_INT_INFO + | bindings::USB_DEVICE_ID_MATCH_DEVICE) as u16, + idVendor: vendor, + idProduct: product, + bInterfaceClass: class, + bInterfaceSubClass: subclass, + bInterfaceProtocol: protocol, + ..pin_init::zeroed() + }) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `usb_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::usb_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info); +} + +/// [`IdTable`](kernel::device_id::IdTable) type for USB. +pub type IdTable = &'static dyn kernel::device_id::IdTable; + +/// Create a USB `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! usb_device_table { + ($($tt:tt)*) => { + $crate::module_device_table!("usb", $crate::usb::DeviceId, $($tt)*); + }; +} + +/// The USB driver trait. +/// +/// # Examples +/// +///``` +/// # use kernel::{bindings, device::Core, usb}; +/// use kernel::prelude::*; +/// +/// struct MyDriver; +/// +/// kernel::usb_device_table!( +/// USB_TABLE, +/// ::IdInfo, +/// [ +/// (usb::DeviceId::from_id(0x1234, 0x5678), ()), +/// (usb::DeviceId::from_id(0xabcd, 0xef01), ()), +/// ] +/// ); +/// +/// impl usb::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const ID_TABLE: usb::IdTable = &USB_TABLE; +/// +/// fn probe<'bound>( +/// _interface: &'bound usb::Interface>, +/// _id: &usb::DeviceId, +/// _info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// Err(ENODEV) +/// } +/// +/// fn disconnect<'bound>( +/// _interface: &'bound usb::Interface>, +/// _data: Pin<&Self::Data<'bound>>, +/// ) { +/// } +/// } +///``` +pub trait Driver { + /// The type holding information about each one of the device ids supported by the driver. + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable; + + /// USB driver probe. + /// + /// Called when a new USB interface is bound to this driver. + /// Implementers should attempt to initialize the interface here. + fn probe<'bound>( + interface: &'bound Interface>, + id: &DeviceId, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// USB driver disconnect. + /// + /// Called when the USB interface is about to be unbound from this driver. + fn disconnect<'bound>( + interface: &'bound Interface>, + data: Pin<&Self::Data<'bound>>, + ); +} + +/// A USB interface. +/// +/// This structure represents the Rust abstraction for a C [`struct usb_interface`]. +/// The implementation abstracts the usage of a C [`struct usb_interface`] passed +/// in from the C side. +/// +/// # Invariants +/// +/// An [`Interface`] instance represents a valid [`struct usb_interface`] created +/// by the C portion of the kernel. +/// +/// [`struct usb_interface`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_interface +#[repr(transparent)] +pub struct Interface( + Opaque, + PhantomData, +); + +impl Interface { + fn as_raw(&self) -> *mut bindings::usb_interface { + self.0.get() + } +} + +// SAFETY: `usb::Interface` is a transparent wrapper of `struct usb_interface`. +// The offset is guaranteed to point to a valid device field inside `usb::Interface`. +unsafe impl device::AsBusDevice for Interface { + const OFFSET: usize = offset_of!(bindings::usb_interface, dev); +} + +// SAFETY: `Interface` is a transparent wrapper of a type that doesn't depend on +// `Interface`'s generic argument. +kernel::impl_device_context_deref!(unsafe { Interface }); +kernel::impl_device_context_into_aref!(Interface); + +impl AsRef> for Interface { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct usb_interface`. + let dev = unsafe { &raw mut ((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl AsRef for Interface { + fn as_ref(&self) -> &Device { + // SAFETY: `self.as_raw()` is valid by the type invariants. + let usb_dev = unsafe { bindings::interface_to_usbdev(self.as_raw()) }; + + // SAFETY: For a valid `struct usb_interface` pointer, the above call to + // `interface_to_usbdev()` guarantees to return a valid pointer to a `struct usb_device`. + unsafe { &*(usb_dev.cast()) } + } +} + +// SAFETY: Instances of `Interface` are always reference-counted. +unsafe impl AlwaysRefCounted for Interface { + #[inline] + fn inc_ref(&self) { + // SAFETY: The invariants of `Interface` guarantee that `self.as_raw()` + // returns a valid `struct usb_interface` pointer, for which we will + // acquire a new refcount. + unsafe { bindings::usb_get_intf(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::usb_put_intf(obj.cast().as_ptr()) } + } +} + +// SAFETY: A `Interface` is always reference-counted and can be released from any thread. +unsafe impl Send for Interface {} + +// SAFETY: It is safe to send a &Interface to another thread because we do not +// allow any mutation through a shared reference. +unsafe impl Sync for Interface {} + +/// A USB device. +/// +/// This structure represents the Rust abstraction for a C [`struct usb_device`]. +/// The implementation abstracts the usage of a C [`struct usb_device`] passed in +/// from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid [`struct usb_device`] created by the C portion of the +/// kernel. +/// +/// [`struct usb_device`]: https://www.kernel.org/doc/html/latest/driver-api/usb/usb.html#c.usb_device +#[repr(transparent)] +struct Device( + Opaque, + PhantomData, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::usb_device { + self.0.get() + } +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + #[inline] + fn inc_ref(&self) { + // SAFETY: The invariants of `Device` guarantee that `self.as_raw()` + // returns a valid `struct usb_device` pointer, for which we will + // acquire a new refcount. + unsafe { bindings::usb_get_dev(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::usb_put_dev(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct usb_device`. + let dev = unsafe { &raw mut ((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: It is safe to send a &Device to another thread because we do not +// allow any mutation through a shared reference. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct usb_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + +/// Declares a kernel module that exposes a single USB driver. +/// +/// # Examples +/// +/// ```ignore +/// module_usb_driver! { +/// type: MyDriver, +/// name: "Module name", +/// author: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_usb_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::usb::Adapter, { $($f)* }); + } +} diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs new file mode 100644 index 000000000..7e253b6f2 --- /dev/null +++ b/rust/kernel/workqueue.rs @@ -0,0 +1,1114 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Work queues. +//! +//! This file has two components: The raw work item API, and the safe work item API. +//! +//! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single +//! type to define multiple `work_struct` fields. This is done by choosing an id for each field, +//! and using that id to specify which field you wish to use. (The actual value doesn't matter, as +//! long as you use different values for different fields of the same struct.) Since these IDs are +//! generic, they are used only at compile-time, so they shouldn't exist in the final binary. +//! +//! # The raw API +//! +//! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an +//! arbitrary function that knows how to enqueue the work item. It should usually not be used +//! directly, but if you want to, you can use it without using the pieces from the safe API. +//! +//! # The safe API +//! +//! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also +//! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user. +//! +//! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type. +//! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue. +//! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something +//! that implements [`WorkItem`]. +//! +//! ## Examples +//! +//! This example defines a struct that holds an integer and can be scheduled on the workqueue. When +//! the struct is executed, it will print the integer. Since there is only one `work_struct` field, +//! we do not need to specify ids for the fields. +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: Work, +//! } +//! +//! impl_has_work! { +//! impl HasWork for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value, +//! work <- new_work!("MyStruct::work"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed. +//! fn print_later(val: Arc) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! # print_later(MyStruct::new(42).unwrap()); +//! ``` +//! +//! The following example shows how multiple `work_struct` fields can be used: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value_1: i32, +//! value_2: i32, +//! #[pin] +//! work_1: Work, +//! #[pin] +//! work_2: Work, +//! } +//! +//! impl_has_work! { +//! impl HasWork for MyStruct { self.work_1 } +//! impl HasWork for MyStruct { self.work_2 } +//! } +//! +//! impl MyStruct { +//! fn new(value_1: i32, value_2: i32) -> Result> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value_1, +//! value_2, +//! work_1 <- new_work!("MyStruct::work_1"), +//! work_2 <- new_work!("MyStruct::work_2"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem<1> for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value_1); +//! } +//! } +//! +//! impl WorkItem<2> for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The second value is: {}\n", this.value_2); +//! } +//! } +//! +//! fn print_1_later(val: Arc) { +//! let _ = workqueue::system().enqueue::, 1>(val); +//! } +//! +//! fn print_2_later(val: Arc) { +//! let _ = workqueue::system().enqueue::, 2>(val); +//! } +//! # print_1_later(MyStruct::new(24, 25).unwrap()); +//! # print_2_later(MyStruct::new(41, 42).unwrap()); +//! ``` +//! +//! This example shows how you can schedule delayed work items: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: DelayedWork, +//! } +//! +//! impl_has_delayed_work! { +//! impl HasDelayedWork for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result> { +//! Arc::pin_init( +//! pin_init!(MyStruct { +//! value, +//! work <- new_delayed_work!("MyStruct::work"), +//! }), +//! GFP_KERNEL, +//! ) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc; +//! +//! fn run(this: Arc) { +//! pr_info!("The value is: {}\n", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed 12 jiffies later. +//! fn print_later(val: Arc) { +//! let _ = workqueue::system().enqueue_delayed(val, 12); +//! } +//! +//! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This +//! /// is equivalent to calling `enqueue_delayed` with a delay of zero. +//! fn print_now(val: Arc) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! # print_later(MyStruct::new(42).unwrap()); +//! # print_now(MyStruct::new(42).unwrap()); +//! ``` +//! +//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) + +use crate::{ + alloc::{AllocError, Flags}, + container_of, + prelude::*, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + Arc, + LockClassKey, // + }, + time::Jiffies, + types::Opaque, +}; +use core::{marker::PhantomData, ptr::NonNull}; + +/// Creates a [`Work`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_work { + ($($name:literal)?) => { + $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_work; + +/// Creates a [`DelayedWork`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_delayed_work { + () => { + $crate::workqueue::DelayedWork::new( + $crate::optional_name!(), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!( + ::core::file!(), + ":", + ::core::line!(), + "_timer" + )), + $crate::static_lock_class!(), + ) + }; + ($name:literal) => { + $crate::workqueue::DelayedWork::new( + $crate::c_str!($name), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!($name, "_timer")), + $crate::static_lock_class!(), + ) + }; +} +pub use new_delayed_work; + +/// A kernel work queue. +/// +/// Wraps the kernel's C `struct workqueue_struct`. +/// +/// It allows work items to be queued to run on thread pools managed by the kernel. Several are +/// always available, for example, `system`, `system_highpri`, `system_long`, etc. +#[repr(transparent)] +pub struct Queue(Opaque); + +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Send for Queue {} +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Sync for Queue {} + +impl Queue { + /// Use the provided `struct workqueue_struct` with Rust. + /// + /// # Safety + /// + /// The caller must ensure that the provided raw pointer is not dangling, that it points at a + /// valid workqueue, and that it remains valid until the end of `'a`. + pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { + // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The + // caller promises that the pointer is not dangling. + unsafe { &*ptr.cast::() } + } + + /// Enqueues a work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue(&self, w: W) -> W::EnqueueOutput + where + W: RawWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which + // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this + // closure. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` + // will have returned true. In this case, `__enqueue` promises that the raw pointer will + // stay valid until we call the function pointer in the `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, + queue_ptr, + work_ptr, + ) + }) + } + } + + /// Enqueues a delayed work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue_delayed(&self, w: W, delay: Jiffies) -> W::EnqueueOutput + where + W: RawDelayedWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_delayed_work_on` will dereference the provided raw pointer, + // which is ok because `__enqueue` guarantees that the pointer is valid for the duration of + // this closure, and the safety requirements of `RawDelayedWorkItem` expands this + // requirement to apply to the entire `delayed_work`. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and + // `bindings::queue_delayed_work_on` will have returned true. In this case, `__enqueue` + // promises that the raw pointer will stay valid until we call the function pointer in the + // `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_delayed_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, + queue_ptr, + container_of!(work_ptr, bindings::delayed_work, work), + delay, + ) + }) + } + } + + /// Tries to spawn the given function or closure as a work item. + /// + /// This method can fail because it allocates memory to store the work item. + pub fn try_spawn( + &self, + flags: Flags, + func: T, + ) -> Result<(), AllocError> { + let init = pin_init!(ClosureWork { + work <- new_work!("Queue::try_spawn"), + func: Some(func), + }); + + self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?); + Ok(()) + } +} + +/// A helper type used in [`try_spawn`]. +/// +/// [`try_spawn`]: Queue::try_spawn +#[pin_data] +struct ClosureWork { + #[pin] + work: Work>, + func: Option, +} + +impl WorkItem for ClosureWork { + type Pointer = Pin>; + + fn run(mut this: Pin>) { + if let Some(func) = this.as_mut().project().func.take() { + (func)() + } + } +} + +/// A raw work item. +/// +/// This is the low-level trait that is designed for being as general as possible. +/// +/// The `ID` parameter to this trait exists so that a single type can provide multiple +/// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then +/// you will implement this trait once for each field, using a different id for each field. The +/// actual value of the id is not important as long as you use different ids for different fields +/// of the same struct. (Fields of different structs need not use different ids.) +/// +/// Note that the id is used only to select the right method to call during compilation. It won't be +/// part of the final executable. +/// +/// # Safety +/// +/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`] +/// remain valid for the duration specified in the guarantees section of the documentation for +/// [`__enqueue`]. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +pub unsafe trait RawWorkItem { + /// The return type of [`Queue::enqueue`]. + type EnqueueOutput; + + /// Enqueues this work item on a queue using the provided `queue_work_on` method. + /// + /// # Guarantees + /// + /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a + /// valid `work_struct` for the duration of the call to the closure. If the closure returns + /// true, then it is further guaranteed that the pointer remains valid until someone calls the + /// function pointer stored in the `work_struct`. + /// + /// # Safety + /// + /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. + /// + /// If the work item type is annotated with any lifetimes, then you must not call the function + /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) + /// + /// If the work item type is not [`Send`], then the function pointer must be called on the same + /// thread as the call to `__enqueue`. + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool; +} + +/// A raw delayed work item. +/// +/// # Safety +/// +/// If the `__enqueue` method in the `RawWorkItem` implementation calls the closure, then the +/// provided pointer must point at the `work` field of a valid `delayed_work`, and the guarantees +/// that `__enqueue` provides about accessing the `work_struct` must also apply to the rest of the +/// `delayed_work` struct. +pub unsafe trait RawDelayedWorkItem: RawWorkItem {} + +/// Defines the method that should be called directly when a work item is executed. +/// +/// This trait is implemented by `Pin>`, [`Arc`] and [`ARef`], and +/// is mainly intended to be implemented for smart pointer types. For your own +/// structs, you would implement [`WorkItem`] instead. The [`run`] method on +/// this trait will usually just perform the appropriate `container_of` +/// translation and then call into the [`run`][WorkItem::run] method from the +/// [`WorkItem`] trait. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +/// +/// # Safety +/// +/// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] +/// method of this trait as the function pointer. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +/// [`run`]: WorkItemPointer::run +pub unsafe trait WorkItemPointer: RawWorkItem { + /// Run this work item. + /// + /// # Safety + /// + /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`] + /// where the `queue_work_on` closure returned true, and the pointer must still be valid. + /// + /// [`__enqueue`]: RawWorkItem::__enqueue + unsafe extern "C" fn run(ptr: *mut bindings::work_struct); +} + +/// Defines the method that should be called when this work item is executed. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +pub trait WorkItem { + /// The pointer type that this struct is wrapped in. This will typically be `Arc` or + /// `Pin>`. + type Pointer: WorkItemPointer; + + /// The method that should be called when this work item is executed. + fn run(this: Self::Pointer); +} + +/// Links for a work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. +/// +/// Wraps the kernel's C `struct work_struct`. +/// +/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct Work { + #[pin] + work: Opaque, + _inner: PhantomData, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Send for Work {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Sync for Work {} + +impl Work { + /// Creates a new instance of [`Work`]. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit + where + T: WorkItem, + { + pin_init!(Self { + work <- Opaque::ffi_init(|slot| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + slot, + Some(T::Pointer::run), + false, + name.as_char_ptr(), + key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `work_struct`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + // + // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that + // the compiler does not complain that the `work` field is unused. + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).work)) } + } +} + +/// Declares that a type contains a [`Work`]. +/// +/// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro +/// like this: +/// +/// ```no_run +/// use kernel::workqueue::{impl_has_work, Work}; +/// +/// struct MyWorkItem { +/// work_field: Work, +/// } +/// +/// impl_has_work! { +/// impl HasWork for MyWorkItem { self.work_field } +/// } +/// ``` +/// +/// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` +/// fields by using a different id for each one. +/// +/// # Safety +/// +/// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be +/// true inverses of each other; that is, they must satisfy the following invariants: +/// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`. +/// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work`. +/// +/// [`impl_has_work!`]: crate::impl_has_work +/// [`raw_get_work`]: HasWork::raw_get_work +/// [`work_container_of`]: HasWork::work_container_of +pub unsafe trait HasWork { + /// Returns a pointer to the [`Work`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work; + + /// Returns a pointer to the struct containing the [`Work`] field. + /// + /// # Safety + /// + /// The pointer must point at a [`Work`] field in a struct of type `Self`. + unsafe fn work_container_of(ptr: *mut Work) -> *mut Self; +} + +/// Used to safely implement the [`HasWork`] trait. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_work, Work}; +/// +/// struct MyStruct<'a, T, const N: usize> { +/// work_field: Work, 17>, +/// f: fn(&'a [T; N]), +/// } +/// +/// impl_has_work! { +/// impl{'a, T, const N: usize} HasWork, 17> +/// for MyStruct<'a, T, N> { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_work { + ($(impl$({$($generics:tt)*})? + HasWork<$work_type:ty $(, $id:tt)?> + for $self:ty + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + } + } + + #[inline] + unsafe fn work_container_of( + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, + ) -> *mut Self { + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + unsafe { $crate::container_of!(ptr, Self, $field) } + } + } + )*}; +} +pub use impl_has_work; + +impl_has_work! { + impl{T} HasWork for ClosureWork { self.work } +} + +/// Links for a delayed work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in +/// a delayed manner. +/// +/// Wraps the kernel's C `struct delayed_work`. +/// +/// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct DelayedWork { + #[pin] + dwork: Opaque, + _inner: PhantomData, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Send for DelayedWork {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl Sync for DelayedWork {} + +impl DelayedWork { + /// Creates a new instance of [`DelayedWork`]. + #[inline] + pub fn new( + work_name: &'static CStr, + work_key: Pin<&'static LockClassKey>, + timer_name: &'static CStr, + timer_key: Pin<&'static LockClassKey>, + ) -> impl PinInit + where + T: WorkItem, + { + pin_init!(Self { + dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + core::ptr::addr_of_mut!((*slot).work), + Some(T::Pointer::run), + false, + work_name.as_char_ptr(), + work_key.as_ptr(), + ) + } + + // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because + // the timer is embedded in a `struct delayed_work`, and only ever scheduled via + // the core workqueue code, and configured to run in irqsafe context. + unsafe { + bindings::timer_init_key( + core::ptr::addr_of_mut!((*slot).timer), + Some(bindings::delayed_work_timer_fn), + bindings::TIMER_IRQSAFE, + timer_name.as_char_ptr(), + timer_key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `delayed_work`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let dw: *mut bindings::delayed_work = + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; + // CAST: Work and work_struct have compatible layouts. + wrk.cast() + } +} + +/// Declares that a type contains a [`DelayedWork`]. +/// +/// # Safety +/// +/// The `HasWork` implementation must return a `work_struct` that is stored in the `work` +/// field of a `delayed_work` with the same access rules as the `work_struct`. +pub unsafe trait HasDelayedWork: HasWork {} + +/// Used to safely implement the [`HasDelayedWork`] trait. +/// +/// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] +/// when using this macro. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; +/// +/// struct MyStruct<'a, T, const N: usize> { +/// work_field: DelayedWork, 17>, +/// f: fn(&'a [T; N]), +/// } +/// +/// impl_has_delayed_work! { +/// impl{'a, T, const N: usize} HasDelayedWork, 17> +/// for MyStruct<'a, T, N> { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_delayed_work { + ($(impl$({$($generics:tt)*})? + HasDelayedWork<$work_type:ty $(, $id:tt)?> + for $self:ty + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? + $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} + + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { + #[inline] + unsafe fn raw_get_work( + ptr: *mut Self + ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + }; + + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } + } + + #[inline] + unsafe fn work_container_of( + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, + ) -> *mut Self { + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let delayed_work = unsafe { + $crate::container_of!(ptr, $crate::bindings::delayed_work, work) + }; + + let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = + delayed_work.cast(); + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + unsafe { $crate::container_of!(delayed_work, Self, $field) } + } + } + )*}; +} +pub use impl_has_delayed_work; + +// SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the +// `run` method of this trait as the function pointer because: +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `Arc`. +unsafe impl WorkItemPointer for Arc +where + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let arc = unsafe { Arc::from_raw(ptr) }; + + T::run(arc) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `Arc`, which means that the ref count will be at least 1, +// and we don't drop the `Arc` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for Arc +where + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // Casting between const and mut is not a problem as long as the pointer is a raw pointer. + let ptr = Arc::into_raw(self).cast_mut(); + + // SAFETY: Pointers into an `Arc` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { Arc::from_raw(ptr) }) + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for Arc +where + T: WorkItem, + T: HasDelayedWork, +{ +} + +// SAFETY: TODO. +unsafe impl WorkItemPointer for Pin> +where + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let boxed = unsafe { KBox::from_raw(ptr) }; + // SAFETY: The box was already pinned when it was enqueued. + let pinned = unsafe { Pin::new_unchecked(boxed) }; + + T::run(pinned) + } +} + +// SAFETY: TODO. +unsafe impl RawWorkItem for Pin> +where + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = (); + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily + // remove the `Pin` wrapper. + let boxed = unsafe { Pin::into_inner_unchecked(self) }; + let ptr = KBox::into_raw(boxed); + + // SAFETY: Pointers into a `KBox` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if !queue_work_on(work_ptr) { + // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a + // workqueue. + unsafe { ::core::hint::unreachable_unchecked() } + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for Pin> +where + T: WorkItem, + T: HasDelayedWork, +{ +} + +// SAFETY: Like the `Arc` implementation, the `__enqueue` implementation for +// `ARef` obtains a `work_struct` from the `Work` field using +// `T::raw_get_work`, so the same safety reasoning applies: +// +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `ARef`. +unsafe impl WorkItemPointer for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + + // SAFETY: This computes the pointer that `__enqueue` got from + // `ARef::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + + // SAFETY: The safety contract of `work_container_of` ensures that it + // returns a valid non-null pointer. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: This pointer comes from `ARef::into_raw` and we've been given + // back ownership. + let aref = unsafe { ARef::from_raw(ptr) }; + + T::run(aref) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `ARef`, which means that the ref count will be at least 1, +// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + let ptr = ARef::into_raw(self); + + // SAFETY: Pointers from ARef::into_raw are valid and non-null. + let work_ptr = unsafe { T::raw_get_work(ptr.as_ptr()) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { ARef::from_raw(ptr) }) + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for ARef +where + T: WorkItem, + T: HasDelayedWork, + T: AlwaysRefCounted, +{ +} + +/// Returns the system work queue (`system_wq`). +/// +/// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are +/// users which expect relatively short queue flush time. +/// +/// Callers shouldn't queue work items which can run for too long. +pub fn system() -> &'static Queue { + // SAFETY: `system_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_wq) } +} + +/// Returns the system high-priority work queue (`system_highpri_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which require higher +/// scheduling priority. +pub fn system_highpri() -> &'static Queue { + // SAFETY: `system_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_highpri_wq) } +} + +/// Returns the system work queue for potentially long-running work items (`system_long_wq`). +/// +/// It is similar to the one returned by [`system`] but may host long running work items. Queue +/// flushing might take relatively long. +pub fn system_long() -> &'static Queue { + // SAFETY: `system_long_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_long_wq) } +} + +/// Returns the system unbound work queue (`system_unbound_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as `max_active` limit is not reached and resources are +/// available. +pub fn system_unbound() -> &'static Queue { + // SAFETY: `system_unbound_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_unbound_wq) } +} + +/// Returns the system freezable work queue (`system_freezable_wq`). +/// +/// It is equivalent to the one returned by [`system`] except that it's freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable() -> &'static Queue { + // SAFETY: `system_freezable_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_wq) } +} + +/// Returns the system power-efficient work queue (`system_power_efficient_wq`). +/// +/// It is inclined towards saving power and is converted to "unbound" variants if the +/// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one +/// returned by [`system`]. +pub fn system_power_efficient() -> &'static Queue { + // SAFETY: `system_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } +} + +/// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). +/// +/// It is similar to the one returned by [`system_power_efficient`] except that is freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable_power_efficient() -> &'static Queue { + // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } +} + +/// Returns the system bottom halves work queue (`system_bh_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which +/// need to run from a softirq context. +pub fn system_bh() -> &'static Queue { + // SAFETY: `system_bh_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_wq) } +} + +/// Returns the system bottom halves high-priority work queue (`system_bh_highpri_wq`). +/// +/// It is similar to the one returned by [`system_bh`] but for work items which +/// require higher scheduling priority. +pub fn system_bh_highpri() -> &'static Queue { + // SAFETY: `system_bh_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_highpri_wq) } +} diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs new file mode 100644 index 000000000..987c9c0c2 --- /dev/null +++ b/rust/kernel/xarray.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! XArray abstraction. +//! +//! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) + +use crate::{ + alloc, + bindings, + build_assert::build_assert, + error::{Error, Result}, + ffi::c_void, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, // +}; +use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; +use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; + +/// An array which efficiently maps sparse integer indices to owned objects. +/// +/// This is similar to a [`crate::alloc::kvec::Vec>`], but more efficient when there are +/// holes in the index space, and can be efficiently grown. +/// +/// # Invariants +/// +/// `self.xa` is always an initialized and valid [`bindings::xarray`] whose entries are either +/// `XA_ZERO_ENTRY` or came from `T::into_foreign`. +/// +/// # Examples +/// +/// ```rust +/// use kernel::alloc::KBox; +/// use kernel::xarray::{AllocKind, XArray}; +/// +/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?; +/// +/// let dead = KBox::new(0xdead, GFP_KERNEL)?; +/// let beef = KBox::new(0xbeef, GFP_KERNEL)?; +/// +/// let mut guard = xa.lock(); +/// +/// assert_eq!(guard.get(0), None); +/// +/// assert_eq!(guard.store(0, dead, GFP_KERNEL)?.as_deref(), None); +/// assert_eq!(guard.get(0).copied(), Some(0xdead)); +/// +/// *guard.get_mut(0).unwrap() = 0xffff; +/// assert_eq!(guard.get(0).copied(), Some(0xffff)); +/// +/// assert_eq!(guard.store(0, beef, GFP_KERNEL)?.as_deref().copied(), Some(0xffff)); +/// assert_eq!(guard.get(0).copied(), Some(0xbeef)); +/// +/// guard.remove(0); +/// assert_eq!(guard.get(0), None); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data(PinnedDrop)] +pub struct XArray { + #[pin] + xa: Opaque, + _p: PhantomData, +} + +#[pinned_drop] +impl PinnedDrop for XArray { + fn drop(self: Pin<&mut Self>) { + self.iter().for_each(|ptr| { + let ptr = ptr.as_ptr(); + // SAFETY: `ptr` came from `T::into_foreign`. + // + // INVARIANT: we own the only reference to the array which is being dropped so the + // broken invariant is not observable on function exit. + drop(unsafe { T::from_foreign(ptr) }) + }); + + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_destroy(self.xa.get()) }; + } +} + +/// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior. +pub enum AllocKind { + /// Consider the first element to be at index 0. + Alloc, + /// Consider the first element to be at index 1. + Alloc1, +} + +impl XArray { + /// Creates a new initializer for this type. + pub fn new(kind: AllocKind) -> impl PinInit { + let flags = match kind { + AllocKind::Alloc => bindings::XA_FLAGS_ALLOC, + AllocKind::Alloc1 => bindings::XA_FLAGS_ALLOC1, + }; + pin_init!(Self { + // SAFETY: `xa` is valid while the closure is called. + // + // INVARIANT: `xa` is initialized here to an empty, valid [`bindings::xarray`]. + xa <- Opaque::ffi_init(|xa| unsafe { + bindings::xa_init_flags(xa, flags) + }), + _p: PhantomData, + }) + } + + fn iter(&self) -> impl Iterator> + '_ { + let mut index = 0; + + // SAFETY: `self.xa` is always valid by the type invariant. + iter::once(unsafe { + bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }) + .chain(iter::from_fn(move || { + // SAFETY: `self.xa` is always valid by the type invariant. + Some(unsafe { + bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }) + })) + .map_while(|ptr| NonNull::new(ptr.cast())) + } + + /// Attempts to lock the [`XArray`] for exclusive access. + pub fn try_lock(&self) -> Option> { + // SAFETY: `self.xa` is always valid by the type invariant. + if (unsafe { bindings::xa_trylock(self.xa.get()) } != 0) { + Some(Guard { + xa: self, + _not_send: NotThreadSafe, + }) + } else { + None + } + } + + /// Locks the [`XArray`] for exclusive access. + pub fn lock(&self) -> Guard<'_, T> { + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + Guard { + xa: self, + _not_send: NotThreadSafe, + } + } +} + +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, T: ForeignOwnable> { + xa: &'a XArray, + _not_send: NotThreadSafe, +} + +impl Drop for Guard<'_, T> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::xa_unlock(self.xa.xa.get()) }; + } +} + +/// The error returned by [`store`](Guard::store). +/// +/// Contains the underlying error and the value that was not stored. +pub struct StoreError { + /// The error that occurred. + pub error: Error, + /// The value that was not stored. + pub value: T, +} + +impl From> for Error { + #[inline] + fn from(value: StoreError) -> Self { + value.error + } +} + +impl<'a, T: ForeignOwnable> Guard<'a, T> { + fn load(&self, index: usize, f: F) -> Option + where + F: FnOnce(NonNull) -> U, + { + // SAFETY: `self.xa.xa` is always valid by the type invariant. + let ptr = unsafe { bindings::xa_load(self.xa.xa.get(), index) }; + let ptr = NonNull::new(ptr.cast())?; + Some(f(ptr)) + } + + /// Provides a reference to the element at the given index. + pub fn get(&self, index: usize) -> Option> { + self.load(index, |ptr| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { T::borrow(ptr.as_ptr()) } + }) + } + + /// Provides a mutable reference to the element at the given index. + pub fn get_mut(&mut self, index: usize) -> Option> { + self.load(index, |ptr| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { T::borrow_mut(ptr.as_ptr()) } + }) + } + + /// Removes and returns the element at the given index. + pub fn remove(&mut self, index: usize) -> Option { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock. + let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast(); + // SAFETY: + // - `ptr` is either NULL or came from `T::into_foreign`. + // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`] + // borrowed from `self` have ended. + unsafe { T::try_from_foreign(ptr) } + } + + /// Stores an element at the given index. + /// + /// May drop the lock if needed to allocate memory, and then reacquire it afterwards. + /// + /// On success, returns the element which was previously at the given index. + /// + /// On failure, returns the element which was attempted to be stored. + pub fn store( + &mut self, + index: usize, + value: T, + gfp: alloc::Flags, + ) -> Result, StoreError> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let new = value.into_foreign(); + + let old = { + let new = new.cast(); + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock. + // + // INVARIANT: `new` came from `T::into_foreign`. + unsafe { bindings::__xa_store(self.xa.xa.get(), index, new, gfp.as_raw()) } + }; + + // SAFETY: `__xa_store` returns the old entry at this index on success or `xa_err` if an + // error happened. + let errno = unsafe { bindings::xa_err(old) }; + if errno != 0 { + // SAFETY: `new` came from `T::into_foreign` and `__xa_store` does not take + // ownership of the value on error. + let value = unsafe { T::from_foreign(new) }; + Err(StoreError { + value, + error: Error::from_errno(errno), + }) + } else { + let old = old.cast(); + // SAFETY: `ptr` is either NULL or came from `T::into_foreign`. + // + // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray + // API; such entries present as `NULL`. + Ok(unsafe { T::try_from_foreign(old) }) + } + } +} + +// SAFETY: `XArray` has no shared mutable state so it is `Send` iff `T` is `Send`. +unsafe impl Send for XArray {} + +// SAFETY: `XArray` serialises the interior mutability it provides so it is `Sync` iff `T` is +// `Send`. +unsafe impl Sync for XArray {} diff --git a/rust/macros/concat_idents.rs b/rust/macros/concat_idents.rs new file mode 100644 index 000000000..47b6add37 --- /dev/null +++ b/rust/macros/concat_idents.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::{ + Ident, + TokenStream, + TokenTree, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + Result, + Token, // +}; + +pub(crate) struct Input { + a: Ident, + _comma: Token![,], + b: Ident, +} + +impl Parse for Input { + fn parse(input: ParseStream<'_>) -> Result { + Ok(Self { + a: input.parse()?, + _comma: input.parse()?, + b: input.parse()?, + }) + } +} + +pub(crate) fn concat_idents(Input { a, b, .. }: Input) -> TokenStream { + let res = Ident::new(&format!("{a}{b}"), b.span()); + TokenStream::from_iter([TokenTree::Ident(res)]) +} diff --git a/rust/macros/export.rs b/rust/macros/export.rs new file mode 100644 index 000000000..6d53521f6 --- /dev/null +++ b/rust/macros/export.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::TokenStream; +use quote::quote; + +/// Please see [`crate::export`] for documentation. +pub(crate) fn export(f: syn::ItemFn) -> TokenStream { + let name = &f.sig.ident; + + quote! { + // This verifies that the function has the same signature as the declaration generated by + // bindgen. It makes use of the fact that all branches of an if/else must have the same + // type. + const _: () = { + if true { + ::kernel::bindings::#name + } else { + #name + }; + }; + + #[no_mangle] + #f + } +} diff --git a/rust/macros/fmt.rs b/rust/macros/fmt.rs new file mode 100644 index 000000000..ce6c72493 --- /dev/null +++ b/rust/macros/fmt.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::collections::BTreeSet; + +use proc_macro2::{Ident, TokenStream, TokenTree}; +use quote::quote_spanned; + +/// Please see [`crate::fmt`] for documentation. +pub(crate) fn fmt(input: TokenStream) -> TokenStream { + let mut input = input.into_iter(); + + let first_opt = input.next(); + let first_owned_str; + let mut names = BTreeSet::new(); + let first_span = { + let Some((mut first_str, first_span)) = (match first_opt.as_ref() { + Some(TokenTree::Literal(first_lit)) => { + first_owned_str = first_lit.to_string(); + Some(first_owned_str.as_str()).and_then(|first| { + let first = first.strip_prefix('"')?; + let first = first.strip_suffix('"')?; + Some((first, first_lit.span())) + }) + } + _ => None, + }) else { + return first_opt.into_iter().chain(input).collect(); + }; + + // Parse `identifier`s from the format string. + // + // See https://doc.rust-lang.org/std/fmt/index.html#syntax. + while let Some((_, rest)) = first_str.split_once('{') { + first_str = rest; + if let Some(rest) = first_str.strip_prefix('{') { + first_str = rest; + continue; + } + if let Some((name, rest)) = first_str.split_once('}') { + first_str = rest; + let name = name.split_once(':').map_or(name, |(name, _)| name); + if !name.is_empty() && !name.chars().all(|c| c.is_ascii_digit()) { + names.insert(name); + } + } + } + first_span + }; + + let adapter = quote_spanned!(first_span => ::kernel::fmt::Adapter); + + let mut args = TokenStream::from_iter(first_opt); + { + let mut flush = |args: &mut TokenStream, current: &mut TokenStream| { + let current = std::mem::take(current); + if !current.is_empty() { + let (lhs, rhs) = (|| { + let mut current = current.into_iter(); + let mut acc = TokenStream::new(); + while let Some(tt) = current.next() { + // Split on `=` only once to handle cases like `a = b = c`. + if matches!(&tt, TokenTree::Punct(p) if p.as_char() == '=') { + names.remove(acc.to_string().as_str()); + // Include the `=` itself to keep the handling below uniform. + acc.extend([tt]); + return (Some(acc), current.collect::()); + } + acc.extend([tt]); + } + (None, acc) + })(); + args.extend(quote_spanned!(first_span => #lhs #adapter(&(#rhs)))); + } + }; + + let mut current = TokenStream::new(); + for tt in input { + match &tt { + TokenTree::Punct(p) if p.as_char() == ',' => { + flush(&mut args, &mut current); + &mut args + } + _ => &mut current, + } + .extend([tt]); + } + flush(&mut args, &mut current); + } + + for name in names { + let name = Ident::new(name, first_span); + args.extend(quote_spanned!(first_span => , #name = #adapter(&#name))); + } + + quote_spanned!(first_span => ::core::format_args!(#args)) +} diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs new file mode 100644 index 000000000..4372cbad3 --- /dev/null +++ b/rust/macros/for_lt.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{ + Span, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + visit::Visit, + visit_mut::VisitMut, + Lifetime, + Result, + Token, + Type, // +}; + +pub(crate) enum HigherRankedType { + Explicit { + _for_token: Token![for], + _lt_token: Token![<], + lifetime: Lifetime, + _gt_token: Token![>], + ty: Type, + }, + Implicit { + ty: Type, + }, +} + +impl Parse for HigherRankedType { + fn parse(input: ParseStream<'_>) -> Result { + if input.peek(Token![for]) { + Ok(Self::Explicit { + _for_token: input.parse()?, + _lt_token: input.parse()?, + lifetime: input.parse()?, + _gt_token: input.parse()?, + ty: input.parse()?, + }) + } else { + Ok(Self::Implicit { ty: input.parse()? }) + } + } +} + +trait TypeExt { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type; + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type; + fn has_lifetime(&self, lt: &Lifetime) -> bool; +} + +impl TypeExt for Type { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type { + struct ElidedLifetimeExpander<'a>(&'a Lifetime); + + impl VisitMut for ElidedLifetimeExpander<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + // Expand explicit `'_` + if lifetime.ident == "_" { + *lifetime = self.0.clone(); + } + } + + fn visit_type_reference_mut(&mut self, reference: &mut syn::TypeReference) { + syn::visit_mut::visit_type_reference_mut(self, reference); + + if reference.lifetime.is_none() { + reference.lifetime = Some(self.0.clone()); + } + } + } + + let mut ret = self.clone(); + ElidedLifetimeExpander(explicit_lt).visit_type_mut(&mut ret); + ret + } + + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type { + struct LifetimeReplacer<'a>(&'a Lifetime, &'a Lifetime); + + impl VisitMut for LifetimeReplacer<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + if lifetime.ident == self.0.ident { + *lifetime = self.1.clone(); + } + } + } + + let mut ret = self.clone(); + LifetimeReplacer(src, dst).visit_type_mut(&mut ret); + ret + } + + fn has_lifetime(&self, lt: &Lifetime) -> bool { + struct HasLifetime<'a>(&'a Lifetime, bool); + + impl Visit<'_> for HasLifetime<'_> { + fn visit_lifetime(&mut self, lifetime: &Lifetime) { + if lifetime.ident == self.0.ident { + self.1 = true; + } + } + + // Macro invocations are opaque; conservatively assume they may + // reference the lifetime. + fn visit_macro(&mut self, _: &syn::Macro) { + self.1 = true; + } + } + + let mut visitor = HasLifetime(lt, false); + visitor.visit_type(self); + visitor.1 + } +} + +struct Prover<'a>(&'a Lifetime, Vec<&'a Type>); + +impl<'a> Prover<'a> { + /// Prove that `ty` is covariant over `'lt`. + /// + /// This also needs to prove that it'll be wellformed for any instance of `'lt`. + /// It can be assumed that `ty` will be wellformed if `'lt` is substituted to `'static`. + fn prove(&mut self, ty: &'a Type) { + match ty { + Type::Paren(ty) => self.prove(&ty.elem), + Type::Group(ty) => self.prove(&ty.elem), + + // No lifetime involved + Type::Never(_) => {} + + // `[T; N]` and `[T]` is covariant over `T`. + Type::Array(ty) => self.prove(&ty.elem), + Type::Slice(ty) => self.prove(&ty.elem), + + Type::Tuple(ty) => { + for elem in &ty.elems { + self.prove(elem); + } + } + + // `*const T` is covariant over `T` + Type::Ptr(ty) if ty.const_token.is_some() => self.prove(&ty.elem), + + // `&T` is covariant over `T` and lifetime. + // + // Note that if we encounter `&'other_lt T`, then we still need to make sure the type + // is wellformed if `T` involves `&'lt`, so we defer to the compiler. + // + // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the + // presence of the type implies `'a: 'static` but this is unsound. + Type::Reference(ty) + if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => + { + self.prove(&ty.elem) + } + + // `&[mut] T` is covariant over lifetime. + // In case we have `&[mut] NoLifetime`, we don't need to do additional checks. + Type::Reference(ty) if !ty.elem.has_lifetime(self.0) => (), + + // No mention of lifetime at all, no need to perform compiler check. + ty if !ty.has_lifetime(self.0) => (), + + // Otherwise, we need to emit checks so that compiler can determine if the types are + // actually covariant. + ty => self.1.push(ty), + } + } +} + +/// Shared implementation for both `ForLt!` and `CovariantForLt!`. +/// +/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime +/// instances (workaround for ). `CovariantForLt!` +/// additionally emits covariance proof functions and sets `N = 1`. +fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream { + let (ty, lifetime) = match input { + HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), + HigherRankedType::Implicit { ty } => { + // If there's no explicit `for<'a>` binder, inject a synthetic `'__elided` lifetime + // and expand elided sites. + let lifetime = Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("__elided", span = Span::mixed_site()), + }; + (ty.expand_elided_lifetime(&lifetime), lifetime) + } + }; + + let mut prover = Prover(&lifetime, Vec::new()); + prover.prove(&ty); + + let mut proof = Vec::new(); + + // Emit proofs for every type that requires additional compiler help in proving covariance. + for (idx, required_proof) in prover.1.into_iter().enumerate() { + // Insert a proof that the type is well-formed. + // + // This is intended to workaround a Rust compiler soundness bug related to HRTB. + // https://github.com/rust-lang/rust/issues/152489 + // + // This needs to be a struct instead of fn to avoid the implied WF bounds. + let wf_proof_name = format_ident!("ProveWf{idx}"); + proof.push(quote!( + struct #wf_proof_name<#lifetime>( + ::core::marker::PhantomData<&#lifetime ()>, #required_proof + ); + )); + + // Insert a proof that the type is covariant. + if prove_covariance { + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } + } + + // Make sure that the type is wellformed when substituting lifetime with `'static`. + // + // Currently the Rust compiler doesn't check this, see the above `ProveWf` documentation. + // + // We prefer to use this way of proving WF-ness as it can work when generics are involved. + let ty_static = ty.replace_lifetime( + &lifetime, + &Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("static"), + }, + ); + + let n: usize = prove_covariance.into(); + + quote!( + ::kernel::types::for_lt::UnsafeForLtImpl::< + dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, + #ty_static, + { + #(#proof)* + + #n + } + > + ) +} + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, false) +} + +pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, true) +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs new file mode 100644 index 000000000..d18fbf4da --- /dev/null +++ b/rust/macros/helpers.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + Attribute, + Error, + LitStr, + Result, // +}; + +/// A string literal that is required to have ASCII value only. +pub(crate) struct AsciiLitStr(LitStr); + +impl Parse for AsciiLitStr { + fn parse(input: ParseStream<'_>) -> Result { + let s: LitStr = input.parse()?; + if !s.value().is_ascii() { + return Err(Error::new_spanned(s, "expected ASCII-only string literal")); + } + Ok(Self(s)) + } +} + +impl ToTokens for AsciiLitStr { + fn to_tokens(&self, ts: &mut TokenStream) { + self.0.to_tokens(ts); + } +} + +impl AsciiLitStr { + pub(crate) fn value(&self) -> String { + self.0.value() + } +} + +pub(crate) fn file() -> String { + #[cfg(not(CONFIG_RUSTC_HAS_SPAN_FILE))] + { + proc_macro::Span::call_site() + .source_file() + .path() + .to_string_lossy() + .into_owned() + } + + #[cfg(CONFIG_RUSTC_HAS_SPAN_FILE)] + { + proc_macro::Span::call_site().file() + } +} + +/// Obtain all `#[cfg]` attributes. +pub(crate) fn gather_cfg_attrs(attr: &[Attribute]) -> impl Iterator + '_ { + attr.iter().filter(|a| a.path().is_ident("cfg")) +} diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs new file mode 100644 index 000000000..ae20ed676 --- /dev/null +++ b/rust/macros/kunit.rs @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Procedural macro to run KUnit tests using a user-space like syntax. +//! +//! Copyright (c) 2023 José Expósito + +use std::ffi::CString; + +use proc_macro2::TokenStream; +use quote::{ + format_ident, + quote, + ToTokens, // +}; +use syn::{ + parse_quote, + Error, + Ident, + Item, + ItemMod, + LitCStr, + Result, // +}; + +pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result { + if test_suite.to_string().len() > 255 { + return Err(Error::new_spanned( + test_suite, + "test suite names cannot exceed the maximum length of 255 bytes", + )); + } + + // We cannot handle modules that defer to another file (e.g. `mod foo;`). + let Some((module_brace, module_items)) = module.content.take() else { + Err(Error::new_spanned( + module, + "`#[kunit_tests(test_name)]` attribute should only be applied to inline modules", + ))? + }; + + // Make the entire module gated behind `CONFIG_KUNIT`. + module + .attrs + .insert(0, parse_quote!(#[cfg(CONFIG_KUNIT="y")])); + + let mut processed_items = Vec::new(); + let mut test_cases = Vec::new(); + + // Generate the test KUnit test suite and a test case for each `#[test]`. + // + // The code generated for the following test module: + // + // ``` + // #[kunit_tests(kunit_test_suit_name)] + // mod tests { + // #[test] + // fn foo() { + // assert_eq!(1, 1); + // } + // + // #[test] + // fn bar() { + // assert_eq!(2, 2); + // } + // } + // ``` + // + // Looks like: + // + // ``` + // unsafe extern "C" fn kunit_rust_wrapper_foo(_test: *mut ::kernel::bindings::kunit) { foo(); } + // unsafe extern "C" fn kunit_rust_wrapper_bar(_test: *mut ::kernel::bindings::kunit) { bar(); } + // + // static mut TEST_CASES: [::kernel::bindings::kunit_case; 3] = [ + // ::kernel::kunit::kunit_case(c"foo", kunit_rust_wrapper_foo), + // ::kernel::kunit::kunit_case(c"bar", kunit_rust_wrapper_bar), + // ::pin_init::zeroed(), + // ]; + // + // ::kernel::kunit_unsafe_test_suite!(kunit_test_suit_name, TEST_CASES); + // ``` + // + // Non-function items (e.g. imports) are preserved. + for item in module_items { + let Item::Fn(mut f) = item else { + processed_items.push(item); + continue; + }; + + if f.attrs + .extract_if(.., |attr| attr.path().is_ident("test")) + .count() + == 0 + { + processed_items.push(Item::Fn(f)); + continue; + } + + let test = f.sig.ident.clone(); + + // Retrieve `#[cfg]` applied on the function which needs to be present on derived items too. + let cfg_attrs: Vec<_> = f + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .cloned() + .collect(); + + // Before the test, override usual `assert!` and `assert_eq!` macros with ones that call + // KUnit instead. + let test_str = test.to_string(); + let path = CString::new(crate::helpers::file()).expect("file path cannot contain NUL"); + processed_items.push(parse_quote! { + #[allow(unused)] + macro_rules! assert { + ($cond:expr $(,)?) => {{ + kernel::kunit_assert!(#test_str, #path, 0, $cond); + }} + } + }); + processed_items.push(parse_quote! { + #[allow(unused)] + macro_rules! assert_eq { + ($left:expr, $right:expr $(,)?) => {{ + kernel::kunit_assert_eq!(#test_str, #path, 0, $left, $right); + }} + } + }); + + // Add back the test item. + processed_items.push(Item::Fn(f)); + + let kunit_wrapper_fn_name = format_ident!("kunit_rust_wrapper_{test}"); + let test_cstr = LitCStr::new( + &CString::new(test_str.as_str()).expect("identifier cannot contain NUL"), + test.span(), + ); + processed_items.push(parse_quote! { + unsafe extern "C" fn #kunit_wrapper_fn_name(_test: *mut ::kernel::bindings::kunit) { + (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SKIPPED; + + // Append any `cfg` attributes the user might have written on their tests so we + // don't attempt to call them when they are `cfg`'d out. An extra `use` is used + // here to reduce the length of the assert message. + #(#cfg_attrs)* + { + (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SUCCESS; + use ::kernel::kunit::is_test_result_ok; + assert!(is_test_result_ok(#test())); + } + } + }); + + test_cases.push(quote!( + ::kernel::kunit::kunit_case(#test_cstr, #kunit_wrapper_fn_name) + )); + } + + let num_tests_plus_1 = test_cases.len() + 1; + processed_items.push(parse_quote! { + static mut TEST_CASES: [::kernel::bindings::kunit_case; #num_tests_plus_1] = [ + #(#test_cases,)* + ::pin_init::zeroed(), + ]; + }); + processed_items.push(parse_quote! { + ::kernel::kunit_unsafe_test_suite!(#test_suite, TEST_CASES); + }); + + module.content = Some((module_brace, processed_items)); + Ok(module.to_token_stream()) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs new file mode 100644 index 000000000..24f96feae --- /dev/null +++ b/rust/macros/lib.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Crate for all kernel procedural macros. + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +// Stable since Rust 1.87.0. +#![feature(extract_if)] +// +// Stable since Rust 1.88.0 under a different name, `proc_macro_span_file`, +// which was added in Rust 1.88.0. This is why `cfg_attr` is used here, i.e. +// to avoid depending on the full `proc_macro_span` on Rust >= 1.88.0. +#![cfg_attr(not(CONFIG_RUSTC_HAS_SPAN_FILE), feature(proc_macro_span))] + +mod concat_idents; +mod export; +mod fmt; +mod for_lt; +mod helpers; +mod kunit; +mod module; +mod paste; +mod vtable; + +use proc_macro::TokenStream; + +use syn::parse_macro_input; + +/// Declares a kernel module. +/// +/// The `type` argument should be a type which implements the [`Module`] +/// trait. Also accepts various forms of kernel metadata. +/// +/// The `params` field describe module parameters. Each entry has the form +/// +/// ```ignore +/// parameter_name: type { +/// default: default_value, +/// description: "Description", +/// } +/// ``` +/// +/// `type` may be one of +/// +/// - [`i8`] +/// - [`u8`] +/// - [`i8`] +/// - [`u8`] +/// - [`i16`] +/// - [`u16`] +/// - [`i32`] +/// - [`u32`] +/// - [`i64`] +/// - [`u64`] +/// - [`isize`] +/// - [`usize`] +/// - [`bool`] +/// +/// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) +/// +/// [`Module`]: ../kernel/trait.Module.html +/// +/// # Examples +/// +/// ```ignore +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyModule, +/// name: "my_kernel_module", +/// authors: ["Rust for Linux Contributors"], +/// description: "My very own kernel module!", +/// license: "GPL", +/// alias: ["alternate_module_name"], +/// params: { +/// my_parameter: i64 { +/// default: 1, +/// description: "This parameter has a default of 1", +/// }, +/// }, +/// } +/// +/// struct MyModule(i32); +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// let foo: i32 = 42; +/// pr_info!("I contain: {}\n", foo); +/// pr_info!("i32 param is: {}\n", module_parameters::my_parameter.read()); +/// Ok(Self(foo)) +/// } +/// } +/// # fn main() {} +/// ``` +/// +/// ## Firmware +/// +/// The following example shows how to declare a kernel module that needs +/// to load binary firmware files. You need to specify the file names of +/// the firmware in the `firmware` field. The information is embedded +/// in the `modinfo` section of the kernel module. For example, a tool to +/// build an initramfs uses this information to put the firmware files into +/// the initramfs image. +/// +/// ``` +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyDeviceDriverModule, +/// name: "my_device_driver_module", +/// authors: ["Rust for Linux Contributors"], +/// description: "My device driver requires firmware", +/// license: "GPL", +/// firmware: ["my_device_firmware1.bin", "my_device_firmware2.bin"], +/// } +/// +/// struct MyDeviceDriverModule; +/// +/// impl kernel::Module for MyDeviceDriverModule { +/// fn init(_module: &'static ThisModule) -> Result { +/// Ok(Self) +/// } +/// } +/// # fn main() {} +/// ``` +/// +/// # Supported argument types +/// - `type`: type which implements the [`Module`] trait (required). +/// - `name`: ASCII string literal of the name of the kernel module (required). +/// - `authors`: array of ASCII string literals of the authors of the kernel module. +/// - `description`: string literal of the description of the kernel module. +/// - `license`: ASCII string literal of the license of the kernel module (required). +/// - `alias`: array of ASCII string literals of the alias names of the kernel module. +/// - `firmware`: array of ASCII string literals of the firmware files of +/// the kernel module. +#[proc_macro] +pub fn module(input: TokenStream) -> TokenStream { + module::module(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Declares or implements a vtable trait. +/// +/// Linux's use of pure vtables is very close to Rust traits, but they differ +/// in how unimplemented functions are represented. In Rust, traits can provide +/// default implementation for all non-required methods (and the default +/// implementation could just return `Error::EINVAL`); Linux typically use C +/// `NULL` pointers to represent these functions. +/// +/// This attribute closes that gap. A trait can be annotated with the +/// `#[vtable]` attribute. Implementers of the trait will then also have to +/// annotate the trait with `#[vtable]`. This attribute generates a `HAS_*` +/// associated constant bool for each method in the trait that is set to true if +/// the implementer has overridden the associated method. +/// +/// For a trait method to be optional, it must have a default implementation. +/// This is also the case for traits annotated with `#[vtable]`, but in this +/// case the default implementation will never be executed. The reason for this +/// is that the functions will be called through function pointers installed in +/// C side vtables. When an optional method is not implemented on a `#[vtable]` +/// trait, a `NULL` entry is installed in the vtable. Thus the default +/// implementation is never called. Since these traits are not designed to be +/// used on the Rust side, it should not be possible to call the default +/// implementation. This is done to ensure that we call the vtable methods +/// through the C vtable, and not through the Rust vtable. Therefore, the +/// default implementation should call `build_error!`, which prevents +/// calls to this function at compile time: +/// +/// ```compile_fail +/// # // Intentionally missing `use`s to simplify `rusttest`. +/// build_error!(VTABLE_DEFAULT_ERROR) +/// ``` +/// +/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`]. +/// +/// This macro should not be used when all functions are required. +/// +/// Additionally, this macro automatically handles the `OwnerModule` +/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;` +/// is added as a required associated type if not already defined; on the +/// impl side, `type OwnerModule = LocalModule;` is automatically inserted +/// if not explicitly defined. +/// +/// # Examples +/// +/// ``` +/// use kernel::error::VTABLE_DEFAULT_ERROR; +/// use kernel::prelude::*; +/// +/// # struct LocalModule; +/// # impl kernel::ModuleMetadata for LocalModule { +/// # const NAME: &'static kernel::str::CStr = c"vtable_doctest"; +/// # +/// # // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`. +/// # const THIS_MODULE: kernel::ThisModule = unsafe { +/// # kernel::ThisModule::from_ptr(core::ptr::null_mut()) +/// # }; +/// # } +/// # +/// # fn main() { +/// // Declares a `#[vtable]` trait +/// #[vtable] +/// pub trait Operations: Send + Sync + Sized { +/// fn foo(&self) -> Result<()> { +/// build_error!(VTABLE_DEFAULT_ERROR) +/// } +/// +/// fn bar(&self) -> Result<()> { +/// build_error!(VTABLE_DEFAULT_ERROR) +/// } +/// } +/// +/// struct Foo; +/// +/// // Implements the `#[vtable]` trait +/// #[vtable] +/// impl Operations for Foo { +/// fn foo(&self) -> Result<()> { +/// # Err(EINVAL) +/// // ... +/// } +/// } +/// +/// assert_eq!(::HAS_FOO, true); +/// assert_eq!(::HAS_BAR, false); +/// # } +/// ``` +/// +/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html +#[proc_macro_attribute] +pub fn vtable(attr: TokenStream, input: TokenStream) -> TokenStream { + parse_macro_input!(attr as syn::parse::Nothing); + vtable::vtable(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Export a function so that C code can call it via a header file. +/// +/// Functions exported using this macro can be called from C code using the declaration in the +/// appropriate header file. It should only be used in cases where C calls the function through a +/// header file; cases where C calls into Rust via a function pointer in a vtable (such as +/// `file_operations`) should not use this macro. +/// +/// This macro has the following effect: +/// +/// * Disables name mangling for this function. +/// * Verifies at compile-time that the function signature matches the declaration in the header +/// file. +/// +/// You must declare the signature of the Rust function in a header file that is included by +/// `rust/bindings/bindings_helper.h`. +/// +/// This macro is *not* the same as the C macros `EXPORT_SYMBOL_*`. All Rust symbols are currently +/// automatically exported with `EXPORT_SYMBOL_GPL`. +#[proc_macro_attribute] +pub fn export(attr: TokenStream, input: TokenStream) -> TokenStream { + parse_macro_input!(attr as syn::parse::Nothing); + export::export(parse_macro_input!(input)).into() +} + +/// Like [`core::format_args!`], but automatically wraps arguments in [`kernel::fmt::Adapter`]. +/// +/// This macro allows generating `fmt::Arguments` while ensuring that each argument is wrapped with +/// `::kernel::fmt::Adapter`, which customizes formatting behavior for kernel logging. +/// +/// Named arguments used in the format string (e.g. `{foo}`) are detected and resolved from local +/// bindings. All positional and named arguments are automatically wrapped. +/// +/// This macro is an implementation detail of other kernel logging macros like [`pr_info!`] and +/// should not typically be used directly. +/// +/// [`kernel::fmt::Adapter`]: ../kernel/fmt/struct.Adapter.html +/// [`pr_info!`]: ../kernel/macro.pr_info.html +#[proc_macro] +pub fn fmt(input: TokenStream) -> TokenStream { + fmt::fmt(input.into()).into() +} + +/// Concatenate two identifiers. +/// +/// This is useful in macros that need to declare or reference items with names +/// starting with a fixed prefix and ending in a user specified name. The resulting +/// identifier has the span of the second argument. +/// +/// # Examples +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// use kernel::macros::concat_idents; +/// +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// $(pub(crate) const $newname: u32 = concat_idents!($prefix, $newname);)+ +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK); +/// ``` +#[proc_macro] +pub fn concat_idents(input: TokenStream) -> TokenStream { + concat_idents::concat_idents(parse_macro_input!(input)).into() +} + +/// Paste identifiers together. +/// +/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a +/// single identifier. +/// +/// This is similar to the [`paste`] crate, but with pasting feature limited to identifiers and +/// literals (lifetimes and documentation strings are not supported). There is a difference in +/// supported modifiers as well. +/// +/// # Examples +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// ::kernel::macros::paste! { +/// $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Modifiers +/// +/// For each identifier, it is possible to attach one or multiple modifiers to +/// it. +/// +/// Currently supported modifiers are: +/// * `span`: change the span of concatenated identifier to the span of the specified token. By +/// default the span of the `[< >]` group is used. +/// * `lower`: change the identifier to lower case. +/// * `upper`: change the identifier to upper case. +/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// ::kernel::macros::paste! { +/// $(pub(crate) const fn [<$newname:lower:span>]() -> u32 { [<$prefix $newname:span>] })+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(br_ok(), binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Literals +/// +/// Literals can also be concatenated with other identifiers: +/// +/// ``` +/// macro_rules! create_numbered_fn { +/// ($name:literal, $val:literal) => { +/// ::kernel::macros::paste! { +/// fn []() -> u32 { $val } +/// } +/// }; +/// } +/// +/// create_numbered_fn!("foo", 100); +/// +/// assert_eq!(some_foo_fn100(), 100) +/// ``` +/// +/// [`paste`]: https://docs.rs/paste/ +#[proc_macro] +pub fn paste(input: TokenStream) -> TokenStream { + let mut tokens = proc_macro2::TokenStream::from(input).into_iter().collect(); + paste::expand(&mut tokens); + tokens + .into_iter() + .collect::() + .into() +} + +/// Registers a KUnit test suite and its test cases using a user-space like syntax. +/// +/// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module +/// is ignored. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::prelude::*; +/// #[kunit_tests(kunit_test_suit_name)] +/// mod tests { +/// #[test] +/// fn foo() { +/// assert_eq!(1, 1); +/// } +/// +/// #[test] +/// fn bar() { +/// assert_eq!(2, 2); +/// } +/// } +/// ``` +#[proc_macro_attribute] +pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { + kunit::kunit_tests(parse_macro_input!(attr), parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + +/// Obtain a type that implements [`ForLt`] for the given higher-ranked type. +/// +/// Please refer to the documentation of the [`ForLt`] trait. +/// +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn ForLt(input: TokenStream) -> TokenStream { + for_lt::for_lt(parse_macro_input!(input)).into() +} + +/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked +/// type. +/// +/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime, +/// providing a safe [`CovariantForLt::cast_ref`] method. +/// +/// Please refer to the documentation of the [`CovariantForLt`] trait. +/// +/// [`CovariantForLt`]: trait.CovariantForLt.html +/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn CovariantForLt(input: TokenStream) -> TokenStream { + for_lt::covariant_for_lt(parse_macro_input!(input)).into() +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs new file mode 100644 index 000000000..bc7027f8d --- /dev/null +++ b/rust/macros/module.rs @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::ffi::CString; + +use proc_macro2::{ + Literal, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + braced, + bracketed, + ext::IdentExt, + parse::{ + Parse, + ParseStream, // + }, + parse_quote, + punctuated::Punctuated, + Error, + Expr, + Ident, + LitStr, + Path, + Result, + Token, + Type, // +}; + +use crate::helpers::*; + +struct ModInfoBuilder<'a> { + module: &'a str, + counter: usize, + ts: TokenStream, + param_ts: TokenStream, +} + +impl<'a> ModInfoBuilder<'a> { + fn new(module: &'a str) -> Self { + ModInfoBuilder { + module, + counter: 0, + ts: TokenStream::new(), + param_ts: TokenStream::new(), + } + } + + fn emit_base(&mut self, field: &str, content: &str, builtin: bool, param: bool) { + let string = if builtin { + // Built-in modules prefix their modinfo strings by `module.`. + format!("{module}.{field}={content}\0", module = self.module) + } else { + // Loadable modules' modinfo strings go as-is. + format!("{field}={content}\0") + }; + let length = string.len(); + let string = Literal::byte_string(string.as_bytes()); + let cfg = if builtin { + quote!(#[cfg(not(MODULE))]) + } else { + quote!(#[cfg(MODULE)]) + }; + + let counter = format_ident!( + "__{module}_{counter}", + module = self.module.to_uppercase(), + counter = self.counter + ); + let item = quote! { + #cfg + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + pub static #counter: [u8; #length] = *#string; + }; + + if param { + self.param_ts.extend(item); + } else { + self.ts.extend(item); + } + + self.counter += 1; + } + + fn emit_only_builtin(&mut self, field: &str, content: &str, param: bool) { + self.emit_base(field, content, true, param) + } + + fn emit_only_loadable(&mut self, field: &str, content: &str, param: bool) { + self.emit_base(field, content, false, param) + } + + fn emit(&mut self, field: &str, content: &str) { + self.emit_internal(field, content, false); + } + + fn emit_internal(&mut self, field: &str, content: &str, param: bool) { + self.emit_only_builtin(field, content, param); + self.emit_only_loadable(field, content, param); + } + + fn emit_param(&mut self, field: &str, param: &str, content: &str) { + let content = format!("{param}:{content}"); + self.emit_internal(field, &content, true); + } + + fn emit_params(&mut self, info: &ModuleInfo) { + let Some(params) = &info.params else { + return; + }; + + for param in params { + let param_name_str = param.name.to_string(); + let param_type_str = param.ptype.to_string(); + + let ops = param_ops_path(¶m_type_str); + + // Note: The spelling of these fields is dictated by the user space + // tool `modinfo`. + self.emit_param("parmtype", ¶m_name_str, ¶m_type_str); + self.emit_param("parm", ¶m_name_str, ¶m.description.value()); + + let static_name = format_ident!("__{}_{}_struct", self.module, param.name); + let param_name_cstr = + CString::new(param_name_str).expect("name contains NUL-terminator"); + let param_name_cstr_with_module = + CString::new(format!("{}.{}", self.module, param.name)) + .expect("name contains NUL-terminator"); + + let param_name = ¶m.name; + let param_type = ¶m.ptype; + let param_default = ¶m.default; + + self.param_ts.extend(quote! { + #[allow(non_upper_case_globals)] + pub(crate) static #param_name: + ::kernel::module_param::ModuleParamAccess<#param_type> = + ::kernel::module_param::ModuleParamAccess::new(#param_default); + + const _: () = { + #[allow(non_upper_case_globals)] + #[link_section = "__param"] + #[used(compiler)] + static #static_name: + ::kernel::module_param::KernelParam = + ::kernel::module_param::KernelParam::new( + ::kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context( + if ::core::cfg!(MODULE) { + #param_name_cstr + } else { + #param_name_cstr_with_module + } + ), + // SAFETY: `__this_module` is constructed by the kernel at load + // time and will not be freed until the module is unloaded. + #[cfg(MODULE)] + mod_: unsafe { + core::ptr::from_ref(&::kernel::bindings::__this_module) + .cast_mut() + }, + #[cfg(not(MODULE))] + mod_: ::core::ptr::null_mut(), + ops: core::ptr::from_ref(&#ops), + perm: 0, // Will not appear in sysfs + level: -1, + flags: 0, + __bindgen_anon_1: ::kernel::bindings::kernel_param__bindgen_ty_1 { + arg: #param_name.as_void_ptr() + }, + } + ); + }; + }); + } + } +} + +fn param_ops_path(param_type: &str) -> Path { + match param_type { + "i8" => parse_quote!(::kernel::module_param::PARAM_OPS_I8), + "u8" => parse_quote!(::kernel::module_param::PARAM_OPS_U8), + "i16" => parse_quote!(::kernel::module_param::PARAM_OPS_I16), + "u16" => parse_quote!(::kernel::module_param::PARAM_OPS_U16), + "i32" => parse_quote!(::kernel::module_param::PARAM_OPS_I32), + "u32" => parse_quote!(::kernel::module_param::PARAM_OPS_U32), + "i64" => parse_quote!(::kernel::module_param::PARAM_OPS_I64), + "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64), + "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE), + "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE), + "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL), + t => panic!("Unsupported parameter type {}", t), + } +} + +/// Parse fields that are required to use a specific order. +/// +/// As fields must follow a specific order, we *could* just parse fields one by one by peeking. +/// However the error message generated when implementing that way is not very friendly. +/// +/// So instead we parse fields in an arbitrary order, but only enforce the ordering after parsing, +/// and if the wrong order is used, the proper order is communicated to the user with error message. +/// +/// Usage looks like this: +/// ```ignore +/// parse_ordered_fields! { +/// from input; +/// +/// // This will extract "foo: " into a variable named "foo". +/// // The variable will have type `Option<_>`. +/// foo => , +/// +/// // If you need the variable name to be different than the key name. +/// // This extracts "baz: " into a variable named "bar". +/// // You might want this if "baz" is a keyword. +/// baz as bar => , +/// +/// // You can mark a key as required, and the variable will no longer be `Option`. +/// // foobar will be of type `Expr` instead of `Option`. +/// foobar [required] => input.parse::()?, +/// } +/// ``` +macro_rules! parse_ordered_fields { + (@gen + [$input:expr] + [$([$name:ident; $key:ident; $parser:expr])*] + [$([$req_name:ident; $req_key:ident])*] + ) => { + $(let mut $name = None;)* + + const EXPECTED_KEYS: &[&str] = &[$(stringify!($key),)*]; + const REQUIRED_KEYS: &[&str] = &[$(stringify!($req_key),)*]; + + let span = $input.span(); + let mut seen_keys = Vec::new(); + + while !$input.is_empty() { + let key = $input.call(Ident::parse_any)?; + + if seen_keys.contains(&key) { + Err(Error::new_spanned( + &key, + format!(r#"duplicated key "{key}". Keys can only be specified once."#), + ))? + } + + $input.parse::()?; + + match &*key.to_string() { + $( + stringify!($key) => $name = Some($parser), + )* + _ => { + Err(Error::new_spanned( + &key, + format!(r#"unknown key "{key}". Valid keys are: {EXPECTED_KEYS:?}."#), + ))? + } + } + + $input.parse::()?; + seen_keys.push(key); + } + + for key in REQUIRED_KEYS { + if !seen_keys.iter().any(|e| e == key) { + Err(Error::new(span, format!(r#"missing required key "{key}""#)))? + } + } + + let mut ordered_keys: Vec<&str> = Vec::new(); + for key in EXPECTED_KEYS { + if seen_keys.iter().any(|e| e == key) { + ordered_keys.push(key); + } + } + + if seen_keys != ordered_keys { + Err(Error::new( + span, + format!(r#"keys are not ordered as expected. Order them like: {ordered_keys:?}."#), + ))? + } + + $(let $req_name = $req_name.expect("required field");)* + }; + + // Handle required fields. + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $key:ident as $name:ident [required] => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)* [$name; $key]] $($rest)* + ) + }; + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $name:ident [required] => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)* [$name; $name]] $($rest)* + ) + }; + + // Handle optional fields. + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $key:ident as $name:ident => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)*] $($rest)* + ) + }; + (@gen + [$input:expr] [$($tok:tt)*] [$($req:tt)*] + $name:ident => $parser:expr, + $($rest:tt)* + ) => { + parse_ordered_fields!( + @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)*] $($rest)* + ) + }; + + (from $input:expr; $($tok:tt)*) => { + parse_ordered_fields!(@gen [$input] [] [] $($tok)*) + } +} + +struct Parameter { + name: Ident, + ptype: Ident, + default: Expr, + description: LitStr, +} + +impl Parse for Parameter { + fn parse(input: ParseStream<'_>) -> Result { + let name = input.parse()?; + input.parse::()?; + let ptype = input.parse()?; + + let fields; + braced!(fields in input); + + parse_ordered_fields! { + from fields; + default [required] => fields.parse()?, + description [required] => fields.parse()?, + } + + Ok(Self { + name, + ptype, + default, + description, + }) + } +} + +pub(crate) struct ModuleInfo { + type_: Type, + license: AsciiLitStr, + name: AsciiLitStr, + authors: Option>, + description: Option, + alias: Option>, + firmware: Option>, + imports_ns: Option>, + params: Option>, +} + +impl Parse for ModuleInfo { + fn parse(input: ParseStream<'_>) -> Result { + parse_ordered_fields!( + from input; + type as type_ [required] => input.parse()?, + name [required] => input.parse()?, + authors => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + description => input.parse()?, + license [required] => input.parse()?, + alias => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + firmware => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + imports_ns => { + let list; + bracketed!(list in input); + Punctuated::parse_terminated(&list)? + }, + params => { + let list; + braced!(list in input); + Punctuated::parse_terminated(&list)? + }, + ); + + Ok(ModuleInfo { + type_, + license, + name, + authors, + description, + alias, + firmware, + imports_ns, + params, + }) + } +} + +pub(crate) fn module(info: ModuleInfo) -> Result { + let ModuleInfo { + type_, + license, + name, + authors, + description, + alias, + firmware, + imports_ns, + params: _, + } = &info; + + // Rust does not allow hyphens in identifiers, use underscore instead. + let ident = name.value().replace('-', "_"); + let mut modinfo = ModInfoBuilder::new(ident.as_ref()); + if let Some(authors) = authors { + for author in authors { + modinfo.emit("author", &author.value()); + } + } + if let Some(description) = description { + modinfo.emit("description", &description.value()); + } + modinfo.emit("license", &license.value()); + if let Some(aliases) = alias { + for alias in aliases { + modinfo.emit("alias", &alias.value()); + } + } + if let Some(firmware) = firmware { + for fw in firmware { + modinfo.emit("firmware", &fw.value()); + } + } + if let Some(imports) = imports_ns { + for ns in imports { + modinfo.emit("import_ns", &ns.value()); + } + } + + // Built-in modules also export the `file` modinfo string. + let file = + std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); + modinfo.emit_only_builtin("file", &file, false); + + modinfo.emit_params(&info); + + let modinfo_ts = modinfo.ts; + let params_ts = modinfo.param_ts; + + let ident_init = format_ident!("__{ident}_init"); + let ident_exit = format_ident!("__{ident}_exit"); + let ident_initcall = format_ident!("__{ident}_initcall"); + let initcall_section = ".initcall6.init"; + + let global_asm = format!( + r#".section "{initcall_section}", "a" + __{ident}_initcall: + .long __{ident}_init - . + .previous + "# + ); + + let name_cstr = CString::new(name.value()).expect("name contains NUL-terminator"); + + Ok(quote! { + /// The module name. + /// + /// Used by the printing macros, e.g. [`info!`]. + const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul(); + + /// The `LocalModule` type is the type of the module created by `module!`, + /// `module_pci_driver!`, `module_platform_driver!`, etc. + type LocalModule = #type_; + + impl ::kernel::ModuleMetadata for #type_ { + const NAME: &'static ::kernel::str::CStr = #name_cstr; + + #[cfg(MODULE)] + const THIS_MODULE: ::kernel::ThisModule = { + extern "C" { + static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; + } + + // SAFETY: `__this_module` is constructed by the kernel at load time + // and lives until the module is unloaded. + unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) } + }; + + #[cfg(not(MODULE))] + const THIS_MODULE: ::kernel::ThisModule = unsafe { + ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) + }; + } + + // Double nested modules, since then nobody can access the public items inside. + #[doc(hidden)] + mod __module_init { + mod __module_init { + use pin_init::PinInit; + + /// The "Rust loadable module" mark. + // + // This may be best done another way later on, e.g. as a new modinfo + // key or a new section. For the moment, keep it simple. + #[cfg(MODULE)] + #[used(compiler)] + static __IS_RUST_MODULE: () = (); + + static mut __MOD: ::core::mem::MaybeUninit = + ::core::mem::MaybeUninit::uninit(); + + // Loadable modules need to export the `{init,cleanup}_module` identifiers. + /// # Safety + /// + /// This function must not be called after module initialization, because it may be + /// freed after that completes. + #[cfg(MODULE)] + #[no_mangle] + #[link_section = ".init.text"] + pub unsafe extern "C" fn init_module() -> ::kernel::ffi::c_int { + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name. + unsafe { __init() } + } + + #[cfg(MODULE)] + #[used(compiler)] + #[link_section = ".init.data"] + static __UNIQUE_ID___addressable_init_module: unsafe extern "C" fn() -> i32 = + init_module; + + #[cfg(MODULE)] + #[no_mangle] + #[link_section = ".exit.text"] + pub extern "C" fn cleanup_module() { + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `init_module` has returned `0` + // (which delegates to `__init`). + unsafe { __exit() } + } + + #[cfg(MODULE)] + #[used(compiler)] + #[link_section = ".exit.data"] + static __UNIQUE_ID___addressable_cleanup_module: extern "C" fn() = cleanup_module; + + // Built-in modules are initialized through an initcall pointer + // and the identifiers need to be unique. + #[cfg(not(MODULE))] + #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] + #[link_section = #initcall_section] + #[used(compiler)] + pub static #ident_initcall: extern "C" fn() -> + ::kernel::ffi::c_int = #ident_init; + + #[cfg(not(MODULE))] + #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] + ::core::arch::global_asm!(#global_asm); + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern "C" fn #ident_init() -> ::kernel::ffi::c_int { + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // placement above in the initcall section. + unsafe { __init() } + } + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern "C" fn #ident_exit() { + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `#ident_init` has + // returned `0` (which delegates to `__init`). + unsafe { __exit() } + } + + /// # Safety + /// + /// This function must only be called once. + unsafe fn __init() -> ::kernel::ffi::c_int { + let initer = ::init( + ::kernel::module::this_module::() + ); + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__exit` cannot be called before or during `__init`. + match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } { + Ok(m) => 0, + Err(e) => e.to_errno(), + } + } + + /// # Safety + /// + /// This function must + /// - only be called once, + /// - be called after `__init` has been called and returned `0`. + unsafe fn __exit() { + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__init` was already called. + unsafe { + // Invokes `drop()` on `__MOD`, which should be used for cleanup. + __MOD.assume_init_drop(); + } + } + + #modinfo_ts + } + } + + mod module_parameters { + #params_ts + } + }) +} diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs new file mode 100644 index 000000000..2181e312a --- /dev/null +++ b/rust/macros/paste.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro2::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; + +fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> { + let mut tokens = tokens.iter(); + let mut segments = Vec::new(); + let mut span = None; + loop { + match tokens.next() { + None => break, + Some(TokenTree::Literal(lit)) => { + // Allow us to concat string literals by stripping quotes + let mut value = lit.to_string(); + if value.starts_with('"') && value.ends_with('"') { + value.remove(0); + value.pop(); + } + segments.push((value, lit.span())); + } + Some(TokenTree::Ident(ident)) => { + let mut value = ident.to_string(); + if value.starts_with("r#") { + value.replace_range(0..2, ""); + } + segments.push((value, ident.span())); + } + Some(TokenTree::Punct(p)) if p.as_char() == ':' => { + let Some(TokenTree::Ident(ident)) = tokens.next() else { + panic!("expected identifier as modifier"); + }; + + let (mut value, sp) = segments.pop().expect("expected identifier before modifier"); + match ident.to_string().as_str() { + // Set the overall span of concatenated token as current span + "span" => { + assert!( + span.is_none(), + "span modifier should only appear at most once" + ); + span = Some(sp); + } + "lower" => value = value.to_lowercase(), + "upper" => value = value.to_uppercase(), + v => panic!("unknown modifier `{v}`"), + }; + segments.push((value, sp)); + } + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => { + let tokens = group.stream().into_iter().collect::>(); + segments.append(&mut concat_helper(tokens.as_slice())); + } + token => panic!("unexpected token in paste segments: {token:?}"), + }; + } + + segments +} + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let segments = concat_helper(tokens); + let pasted: String = segments.into_iter().map(|x| x.0).collect(); + TokenTree::Ident(Ident::new(&pasted, group_span)) +} + +pub(crate) fn expand(tokens: &mut Vec) { + for token in tokens.iter_mut() { + if let TokenTree::Group(group) = token { + let delimiter = group.delimiter(); + let span = group.span(); + let mut stream: Vec<_> = group.stream().into_iter().collect(); + // Find groups that looks like `[< A B C D >]` + if delimiter == Delimiter::Bracket + && stream.len() >= 3 + && matches!(&stream[0], TokenTree::Punct(p) if p.as_char() == '<') + && matches!(&stream[stream.len() - 1], TokenTree::Punct(p) if p.as_char() == '>') + { + // Replace the group with concatenated token + *token = concat(&stream[1..stream.len() - 1], span); + } else { + // Recursively expand tokens inside the group + expand(&mut stream); + let mut group = Group::new(delimiter, stream.into_iter().collect()); + group.set_span(span); + *token = TokenTree::Group(group); + } + } + } + + // Path segments cannot contain invisible delimiter group, so remove them if any. + for i in (0..tokens.len().saturating_sub(3)).rev() { + // Looking for a double colon + if matches!( + (&tokens[i + 1], &tokens[i + 2]), + (TokenTree::Punct(a), TokenTree::Punct(b)) + if a.as_char() == ':' && a.spacing() == Spacing::Joint && b.as_char() == ':' + ) { + match &tokens[i + 3] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i + 3..i + 4, group.stream()); + } + _ => (), + } + + match &tokens[i] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i..i + 1, group.stream()); + } + _ => (), + } + } + } +} diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs new file mode 100644 index 000000000..be9a5ed8a --- /dev/null +++ b/rust/macros/vtable.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 + +use std::{ + collections::HashSet, + iter::Extend, // +}; + +use proc_macro2::{ + Ident, + TokenStream, // +}; +use quote::ToTokens; +use syn::{ + parse_quote, + Error, + ImplItem, + Item, + ItemImpl, + ItemTrait, + Result, + TraitItem, // +}; + +fn handle_trait(mut item: ItemTrait) -> Result { + let mut gen_items = Vec::new(); + + gen_items.push(parse_quote! { + /// A marker to prevent implementors from forgetting to use [`#[vtable]`](vtable) + /// attribute when implementing this trait. + const USE_VTABLE_ATTR: (); + }); + + // Add `type OwnerModule: ModuleMetadata` as a required associated type if + // the trait does not already define it. + if !item + .items + .iter() + .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule")) + { + gen_items.push(parse_quote! { + /// The module implementing this vtable trait. + /// + /// Automatically set to `crate::LocalModule` by the `#[vtable]` + /// impl macro. + type OwnerModule: ::kernel::ModuleMetadata; + }); + } + + for item in &item.items { + if let TraitItem::Fn(fn_item) = item { + let name = &fn_item.sig.ident; + let gen_const_name = Ident::new( + &format!("HAS_{}", name.to_string().to_uppercase()), + name.span(), + ); + + // We don't know on the implementation-site whether a method is required or provided + // so we have to generate a const for all methods. + let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); + let comment = + format!("Indicates if the `{name}` method is overridden by the implementor."); + gen_items.push(parse_quote! { + #(#cfg_attrs)* + #[doc = #comment] + const #gen_const_name: bool = false; + }); + } + } + + item.items.extend(gen_items); + Ok(item) +} + +fn handle_impl(mut item: ItemImpl) -> Result { + let mut gen_items = Vec::new(); + let mut defined_items = HashSet::new(); + + // Iterate over all user-defined items to gather any possible explicit overrides. + for item in &item.items { + match item { + ImplItem::Const(const_item) => { + defined_items.insert(const_item.ident.clone()); + } + ImplItem::Type(type_item) => { + defined_items.insert(type_item.ident.clone()); + } + _ => {} + } + } + + gen_items.push(parse_quote! { + const USE_VTABLE_ATTR: () = (); + }); + + // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined. + // `crate::LocalModule` resolves to the real module type (via `module!`) or a + // dummy fallback in non-module contexts (e.g., doctests). + if !defined_items.contains(&parse_quote!(OwnerModule)) { + gen_items.push(parse_quote! { + type OwnerModule = crate::LocalModule; + }); + } + + for item in &item.items { + if let ImplItem::Fn(fn_item) = item { + let name = &fn_item.sig.ident; + let gen_const_name = Ident::new( + &format!("HAS_{}", name.to_string().to_uppercase()), + name.span(), + ); + // Skip if it's declared already -- this allows user override. + if defined_items.contains(&gen_const_name) { + continue; + } + let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); + gen_items.push(parse_quote! { + #(#cfg_attrs)* + const #gen_const_name: bool = true; + }); + } + } + + item.items.extend(gen_items); + Ok(item) +} + +pub(crate) fn vtable(input: Item) -> Result { + match input { + Item::Trait(item) => Ok(handle_trait(item)?.into_token_stream()), + Item::Impl(item) => Ok(handle_impl(item)?.into_token_stream()), + _ => Err(Error::new_spanned( + input, + "`#[vtable]` attribute should only be applied to trait or impl block", + ))?, + } +} diff --git a/rust/pin-init/CONTRIBUTING.md b/rust/pin-init/CONTRIBUTING.md new file mode 100644 index 000000000..16c899a7a --- /dev/null +++ b/rust/pin-init/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contributing to `pin-init` + +Thanks for showing interest in contributing to `pin-init`! This document outlines the guidelines for +contributing to `pin-init`. + +All contributions are double-licensed under Apache 2.0 and MIT. You can find the respective licenses +in the `LICENSE-APACHE` and `LICENSE-MIT` files. + +## Non-Code Contributions + +### Bug Reports + +For any type of bug report, please submit an issue using the bug report issue template. + +If the issue is a soundness issue, please privately report it as a security vulnerability via the +GitHub web interface. + +### Feature Requests + +If you have any feature requests, please submit an issue using the feature request issue template. + +### Questions and Getting Help + +You can ask questions in the Discussions page of the GitHub repository. If you're encountering +problems or just have questions related to `pin-init` in the Linux kernel, you can also ask your +questions in the [Rust-for-Linux Zulip](https://rust-for-linux.zulipchat.com/) or see +. + +## Contributing Code + +### Linux Kernel + +`pin-init` is used by the Linux kernel and all commits are synchronized to it. For this reason, the +same requirements for commits apply to `pin-init`. See [the kernel's documentation] for details. The +rest of this document will also cover some of the rules listed there and additional ones. + +[the kernel's documentation]: https://docs.kernel.org/process/submitting-patches.html + +Contributions to `pin-init` ideally go through the [GitHub repository], because that repository runs +a CI with lots of tests not present in the kernel. However, patches are also accepted (though not +preferred). Do note that there are some files that are only present in the GitHub repository such as +tests, licenses and cargo related files. Making changes to them can only happen via GitHub. + +[GitHub repository]: https://github.com/Rust-for-Linux/pin-init + +### Commit Style + +Everything must compile without errors or warnings and all tests must pass after **every commit**. +This is important for bisection and also required by the kernel. + +Each commit should be a single, logically cohesive change. Of course it's best to keep the changes +small and digestible, but logically linked changes should be made in the same commit. For example, +when fixing typos, create a single commit that fixes all of them instead of one commit per typo. + +Commits must have a meaningful commit title. Commits with changes to files in the `internal` +directory should have a title prefixed with `internal:`. The commit message should explain the +change and its rationale. You also have to add your `Signed-off-by` tag, see [Developer's +Certificate of Origin]. This has to be done for both mailing list submissions as well as GitHub +submissions. + +[Developer's Certificate of Origin]: https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin + +Any changes made to public APIs must be documented not only in the commit message, but also in the +`CHANGELOG.md` file. This is especially important for breaking changes, as those warrant a major +version bump. + +If you make changes to the top-level crate documentation, you also need to update the `README.md` +via `cargo rdme`. + +Some of these rules can be ignored if the change is done solely to files that are not present in the +kernel version of this library. Those files are documented in the `sync-kernel.sh` script at the +very bottom in the `--exclude` flag given to the `git am` command. diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md new file mode 100644 index 000000000..2312c9e75 --- /dev/null +++ b/rust/pin-init/README.md @@ -0,0 +1,247 @@ +[![Crates.io](https://img.shields.io/crates/v/pin-init.svg)](https://crates.io/crates/pin-init) +[![Documentation](https://docs.rs/pin-init/badge.svg)](https://docs.rs/pin-init/) +[![Dependency status](https://deps.rs/repo/github/Rust-for-Linux/pin-init/status.svg)](https://deps.rs/repo/github/Rust-for-Linux/pin-init) +![License](https://img.shields.io/crates/l/pin-init) +[![Toolchain](https://img.shields.io/badge/toolchain-nightly-red)](#nightly-only) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Rust-for-Linux/pin-init/ci.yml) +# `pin-init` + +> [!NOTE] +> +> This crate was originally named [`pinned-init`], but the migration to +> `pin-init` is not yet complete. The `legacy` branch contains the current +> version of the `pinned-init` crate & the `main` branch already incorporates +> the rename to `pin-init`. +> +> There are still some changes needed on the kernel side before the migration +> can be completed. + +[`pinned-init`]: https://crates.io/crates/pinned-init + + + +Library to safely and fallibly initialize pinned `struct`s using in-place constructors. + +[Pinning][pinning] is Rust's way of ensuring data does not move. + +It also allows in-place initialization of big `struct`s that would otherwise produce a stack +overflow. + +This library's main use-case is in [Rust-for-Linux]. Although this version can be used +standalone. + +There are cases when you want to in-place initialize a struct. For example when it is very big +and moving it from the stack is not an option, because it is bigger than the stack itself. +Another reason would be that you need the address of the object to initialize it. This stands +in direct conflict with Rust's normal process of first initializing an object and then moving +it into it's final memory location. For more information, see +. + +This library allows you to do in-place initialization safely. + +### Nightly Needed for `alloc` feature + +This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +enabled and thus this feature can only be used with a nightly compiler. When enabling the +`alloc` feature, the user will be required to activate `allocator_api` as well. + +[`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html + +The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +mode. + +### Nightly needed for `unsafe-pinned` feature + +This feature enables the `Wrapper` implementation on the unstable `core::pin::UnsafePinned` type. +This requires the [`unsafe_pinned` unstable feature](https://github.com/rust-lang/rust/issues/125735) +and therefore a nightly compiler. Note that this feature is not enabled by default. + +## Overview + +To initialize a `struct` with an in-place constructor you will need two things: +- an in-place constructor, +- a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], + [`Box`] or any other smart pointer that supports this library). + +To get an in-place constructor there are generally three options: +- directly creating an in-place constructor using the [`pin_init!`] macro, +- a custom function/macro returning an in-place constructor provided by someone else, +- using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. + +Aside from pinned initialization, this library also supports in-place construction without +pinning, the macros/types/functions are generally named like the pinned variants without the +`pin_` prefix. + +## Examples + +Throughout the examples we will often make use of the `CMutex` type which can be found in +`../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +requires it to be pinned to be locked and thus is a prime candidate for using this library. + +### Using the [`pin_init!`] macro + +If you want to use [`PinInit`], then you will have to annotate your `struct` with +`#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +[structurally pinned fields]. After doing this, you can then create an in-place constructor via +[`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +that you need to write `<-` instead of `:` for fields that you want to initialize in-place. + +```rust +use pin_init::{pin_data, pin_init, InPlaceInit}; + +#[pin_data] +struct Foo { + #[pin] + a: CMutex, + b: u32, +} + +let foo = pin_init!(Foo { + a <- CMutex::new(42), + b: 24, +}); +``` + +`foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like +(or just the stack) to actually initialize a `Foo`: + +```rust +let foo: Result>, AllocError> = Box::pin_init(foo); +``` + +For more information see the [`pin_init!`] macro. + +### Using a custom function/macro that returns an initializer + +Many types that use this library supply a function/macro that returns an initializer, because +the above method only works for types where you can access the fields. + +```rust +let mtx: Result>>, _> = Arc::pin_init(CMutex::new(42)); +``` + +To declare an init macro/function you just return an [`impl PinInit`]: + +```rust +#[pin_data] +struct DriverData { + #[pin] + status: CMutex, + buffer: Box<[u8; 1_000_000]>, +} + +impl DriverData { + fn new() -> impl PinInit { + pin_init!(Self { + status <- CMutex::new(0), + buffer: Box::init(pin_init::init_zeroed())?, + }? Error) + } +} +``` + +### Manual creation of an initializer + +Often when working with primitives the previous approaches are not sufficient. That is where +[`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +[`impl PinInit`] directly from a closure. Of course you have to ensure that the closure +actually does the initialization in the correct way. Here are the things to look out for +(we are calling the parameter to the closure `slot`): +- when the closure returns `Ok(())`, then it has completed the initialization successfully, so + `slot` now contains a valid bit pattern for the type `T`, +- when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so + you need to take care to clean up anything if your initialization fails mid-way, +- you may assume that `slot` will stay pinned even after the closure returns until `drop` of + `slot` gets called. + +```rust +use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +use core::{ + marker::PhantomPinned, + cell::UnsafeCell, + pin::Pin, + mem::MaybeUninit, +}; +mod bindings { + #[repr(C)] + pub struct foo { + /* fields from C ... */ + } + extern "C" { + pub fn init_foo(ptr: *mut foo); + pub fn destroy_foo(ptr: *mut foo); + #[must_use = "you must check the error return code"] + pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; + } +} + +/// # Invariants +/// +/// `foo` is always initialized +#[pin_data(PinnedDrop)] +pub struct RawFoo { + #[pin] + _p: PhantomPinned, + #[pin] + foo: UnsafeCell>, +} + +impl RawFoo { + pub fn new(flags: u32) -> impl PinInit { + // SAFETY: + // - when the closure returns `Ok(())`, then it has successfully initialized and + // enabled `foo`, + // - when it returns `Err(e)`, then it has cleaned up before + unsafe { + pin_init_from_closure(move |slot: *mut Self| { + // `slot` contains uninit memory, avoid creating a reference. + let foo = &raw mut (*slot).foo; + let foo = UnsafeCell::raw_get(foo).cast::(); + + // Initialize the `foo` + bindings::init_foo(foo); + + // Try to enable it. + let err = bindings::enable_foo(foo, flags); + if err != 0 { + // Enabling has failed, first clean up the foo and then return the error. + bindings::destroy_foo(foo); + Err(err) + } else { + // All fields of `RawFoo` have been initialized, since `_p` is a ZST. + Ok(()) + } + }) + } + } +} + +#[pinned_drop] +impl PinnedDrop for RawFoo { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Since `foo` is initialized, destroying is safe. + unsafe { bindings::destroy_foo(self.foo.get().cast::()) }; + } +} +``` + +For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +the `kernel` crate. The [`sync`] module is a good starting point. + +[`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +[pinning]: https://doc.rust-lang.org/std/pin/index.html +[structurally pinned fields]: https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning +[stack]: https://docs.rs/pin-init/latest/pin_init/macro.stack_pin_init.html +[`impl PinInit`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl PinInit`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl Init`]: https://docs.rs/pin-init/latest/pin_init/trait.Init.html +[Rust-for-Linux]: https://rust-for-linux.com/ + + + + +[`Arc`]: https://doc.rust-lang.org/stable/alloc/sync/struct.Arc.html +[`Box`]: https://doc.rust-lang.org/stable/alloc/boxed/struct.Box.html diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs new file mode 100644 index 000000000..c05139927 --- /dev/null +++ b/rust/pin-init/examples/big_struct_in_place.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use pin_init::*; + +// Struct with size over 1GiB +#[derive(Debug)] +#[allow(dead_code)] +pub struct BigStruct { + buf: [u8; 1024 * 1024 * 1024], + a: u64, + b: u64, + c: u64, + d: u64, + managed_buf: ManagedBuf, +} + +#[derive(Debug)] +pub struct ManagedBuf { + buf: [u8; 1024 * 1024], +} + +impl ManagedBuf { + pub fn new() -> impl Init { + init!(ManagedBuf { buf <- init_zeroed() }) + } +} + +fn main() { + #[cfg(any(feature = "std", feature = "alloc"))] + { + // we want to initialize the struct in-place, otherwise we would get a stackoverflow + let buf: Box = Box::init(init!(BigStruct { + buf <- init_zeroed(), + a: 7, + b: 186, + c: 7789, + d: 34, + managed_buf <- ManagedBuf::new(), + })) + .unwrap(); + println!("{}", core::mem::size_of_val(&*buf)); + } +} diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs new file mode 100644 index 000000000..96f095398 --- /dev/null +++ b/rust/pin-init/examples/error.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::convert::Infallible; + +#[cfg(feature = "alloc")] +use std::alloc::AllocError; + +#[derive(Debug)] +pub struct Error; + +impl From for Error { + #[inline] + fn from(e: Infallible) -> Self { + match e {} + } +} + +#[cfg(feature = "alloc")] +impl From for Error { + #[inline] + fn from(_: AllocError) -> Self { + Self + } +} + +#[allow(dead_code)] +fn main() { + let _ = Error; +} diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs new file mode 100644 index 000000000..424585fe2 --- /dev/null +++ b/rust/pin-init/examples/linked_list.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::Cell, + marker::PhantomPinned, + pin::Pin, + ptr::{self, NonNull}, +}; + +use pin_init::*; + +#[allow(unused_attributes)] +mod error; +#[allow(unused_imports)] +use error::Error; + +#[pin_data(PinnedDrop)] +#[repr(C)] +#[derive(Debug)] +pub struct ListHead { + next: Link, + prev: Link, + #[pin] + pin: PhantomPinned, +} + +impl ListHead { + #[inline] + pub fn new() -> impl PinInit { + pin_init!(&this in Self { + next: unsafe { Link::new_unchecked(this) }, + prev: unsafe { Link::new_unchecked(this) }, + pin: PhantomPinned, + }) + } + + #[inline] + #[allow(dead_code)] + pub fn insert_next(list: &ListHead) -> impl PinInit + '_ { + pin_init!(&this in Self { + prev: list.next.prev().replace(unsafe { Link::new_unchecked(this)}), + next: list.next.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }) + } + + #[inline] + pub fn insert_prev(list: &ListHead) -> impl PinInit + '_ { + pin_init!(&this in Self { + next: list.prev.next().replace(unsafe { Link::new_unchecked(this)}), + prev: list.prev.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }) + } + + #[inline] + pub fn next(&self) -> Option> { + if ptr::eq(self.next.as_ptr(), self) { + None + } else { + Some(unsafe { NonNull::new_unchecked(self.next.as_ptr() as *mut Self) }) + } + } + + #[allow(dead_code)] + pub fn size(&self) -> usize { + let mut size = 1; + let mut cur = self.next.clone(); + while !ptr::eq(self, cur.cur()) { + cur = cur.next().clone(); + size += 1; + } + size + } +} + +#[pinned_drop] +impl PinnedDrop for ListHead { + //#[inline] + fn drop(self: Pin<&mut Self>) { + if !ptr::eq(self.next.as_ptr(), &*self) { + let next = unsafe { &*self.next.as_ptr() }; + let prev = unsafe { &*self.prev.as_ptr() }; + next.prev.set(&self.prev); + prev.next.set(&self.next); + } + } +} + +#[repr(transparent)] +#[derive(Clone, Debug)] +struct Link(Cell>); + +impl Link { + /// # Safety + /// + /// The contents of the pointer should form a consistent circular + /// linked list; for example, a "next" link should be pointed back + /// by the target `ListHead`'s "prev" link and a "prev" link should be + /// pointed back by the target `ListHead`'s "next" link. + #[inline] + unsafe fn new_unchecked(ptr: NonNull) -> Self { + Self(Cell::new(ptr)) + } + + #[inline] + fn next(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).next } + } + + #[inline] + #[allow(dead_code)] + fn prev(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).prev } + } + + #[allow(dead_code)] + fn cur(&self) -> &ListHead { + unsafe { &*self.0.get().as_ptr() } + } + + #[inline] + fn replace(&self, other: Link) -> Link { + unsafe { Link::new_unchecked(self.0.replace(other.0.get())) } + } + + #[inline] + fn as_ptr(&self) -> *const ListHead { + self.0.get().as_ptr() + } + + #[inline] + fn set(&self, val: &Link) { + self.0.set(val.0.get()); + } +} + +#[allow(dead_code)] +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() -> Result<(), Error> { + let a = Box::pin_init(ListHead::new())?; + stack_pin_init!(let b = ListHead::insert_next(&a)); + stack_pin_init!(let c = ListHead::insert_next(&a)); + stack_pin_init!(let d = ListHead::insert_next(&b)); + let e = Box::pin_init(ListHead::insert_next(&b))?; + println!("a ({a:p}): {a:?}"); + println!("b ({b:p}): {b:?}"); + println!("c ({c:p}): {c:?}"); + println!("d ({d:p}): {d:?}"); + println!("e ({e:p}): {e:?}"); + let mut inspect = &*a; + while let Some(next) = inspect.next() { + println!("({inspect:p}): {inspect:?}"); + inspect = unsafe { &*next.as_ptr() }; + if core::ptr::eq(inspect, &*a) { + break; + } + } + Ok(()) +} diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs new file mode 100644 index 000000000..e8d4dbb66 --- /dev/null +++ b/rust/pin-init/examples/mutex.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(clippy::missing_safety_doc)] + +use core::{ + cell::{Cell, UnsafeCell}, + marker::PhantomPinned, + ops::{Deref, DerefMut}, + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, +}; +#[cfg(feature = "std")] +use std::{ + sync::Arc, + thread::{self, sleep, Builder, Thread}, + time::Duration, +}; + +use pin_init::*; +#[allow(unused_attributes)] +#[path = "./linked_list.rs"] +pub mod linked_list; +use linked_list::*; + +pub struct SpinLock { + inner: AtomicBool, +} + +impl SpinLock { + #[inline] + pub fn acquire(&self) -> SpinLockGuard<'_> { + while self + .inner + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + #[cfg(feature = "std")] + while self.inner.load(Ordering::Relaxed) { + thread::yield_now(); + } + } + SpinLockGuard(self) + } + + #[inline] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + inner: AtomicBool::new(false), + } + } +} + +pub struct SpinLockGuard<'a>(&'a SpinLock); + +impl Drop for SpinLockGuard<'_> { + #[inline] + fn drop(&mut self) { + self.0.inner.store(false, Ordering::Release); + } +} + +#[pin_data] +pub struct CMutex { + #[pin] + wait_list: ListHead, + spin_lock: SpinLock, + locked: Cell, + #[pin] + data: UnsafeCell, +} + +impl CMutex { + #[inline] + pub fn new(val: impl PinInit) -> impl PinInit { + pin_init!(CMutex { + wait_list <- ListHead::new(), + spin_lock: SpinLock::new(), + locked: Cell::new(false), + data <- UnsafeCell::pin_init(val), + }) + } + + #[inline] + pub fn lock(&self) -> Pin> { + let mut sguard = self.spin_lock.acquire(); + if self.locked.get() { + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); + // println!("wait list length: {}", self.wait_list.size()); + while self.locked.get() { + drop(sguard); + #[cfg(feature = "std")] + thread::park(); + sguard = self.spin_lock.acquire(); + } + } + self.locked.set(true); + unsafe { + Pin::new_unchecked(CMutexGuard { + mtx: self, + _pin: PhantomPinned, + }) + } + } + + #[allow(dead_code)] + pub fn get_data_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: we have an exclusive reference and thus nobody has access to data. + unsafe { &mut *self.data.get() } + } +} + +unsafe impl Send for CMutex {} +unsafe impl Sync for CMutex {} + +pub struct CMutexGuard<'a, T> { + mtx: &'a CMutex, + _pin: PhantomPinned, +} + +impl Drop for CMutexGuard<'_, T> { + #[inline] + fn drop(&mut self) { + let sguard = self.mtx.spin_lock.acquire(); + self.mtx.locked.set(false); + if let Some(list_field) = self.mtx.wait_list.next() { + let _wait_entry = list_field.as_ptr().cast::(); + #[cfg(feature = "std")] + unsafe { + (*_wait_entry).thread.unpark() + }; + } + drop(sguard); + } +} + +impl Deref for CMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } +} + +impl DerefMut for CMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } +} + +#[pin_data] +#[repr(C)] +struct WaitEntry { + #[pin] + wait_list: ListHead, + #[cfg(feature = "std")] + thread: Thread, +} + +impl WaitEntry { + #[inline] + fn insert_new(list: &ListHead) -> impl PinInit + '_ { + #[cfg(feature = "std")] + { + pin_init!(Self { + thread: thread::current(), + wait_list <- ListHead::insert_prev(list), + }) + } + #[cfg(not(feature = "std"))] + { + pin_init!(Self { + wait_list <- ListHead::insert_prev(list), + }) + } + } +} + +#[cfg_attr(test, test)] +#[allow(dead_code)] +fn main() { + #[cfg(feature = "std")] + { + let mtx: Pin>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = if cfg!(miri) { 100 } else { 1_000 }; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", *mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs new file mode 100644 index 000000000..00f457e68 --- /dev/null +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// inspired by +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +#[cfg(not(windows))] +mod pthread_mtx { + #[cfg(feature = "alloc")] + use core::alloc::AllocError; + use core::{ + cell::UnsafeCell, + marker::PhantomPinned, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + pin::Pin, + }; + use pin_init::*; + use std::convert::Infallible; + + #[pin_data(PinnedDrop)] + pub struct PThreadMutex { + #[pin] + raw: UnsafeCell, + data: UnsafeCell, + #[pin] + pin: PhantomPinned, + } + + unsafe impl Send for PThreadMutex {} + unsafe impl Sync for PThreadMutex {} + + #[pinned_drop] + impl PinnedDrop for PThreadMutex { + fn drop(self: Pin<&mut Self>) { + unsafe { + libc::pthread_mutex_destroy(self.raw.get()); + } + } + } + + #[derive(Debug)] + pub enum Error { + #[allow(dead_code)] + IO(std::io::Error), + #[allow(dead_code)] + Alloc, + } + + impl From for Error { + fn from(e: Infallible) -> Self { + match e {} + } + } + + #[cfg(feature = "alloc")] + impl From for Error { + fn from(_: AllocError) -> Self { + Self::Alloc + } + } + + impl PThreadMutex { + #[allow(dead_code)] + pub fn new(data: T) -> impl PinInit { + fn init_raw() -> impl PinInit, Error> { + let init = |slot: *mut UnsafeCell| { + // we can cast, because `UnsafeCell` has the same layout as T. + let slot: *mut libc::pthread_mutex_t = slot.cast(); + let mut attr = MaybeUninit::uninit(); + let attr = attr.as_mut_ptr(); + // SAFETY: ptr is valid + let ret = unsafe { libc::pthread_mutexattr_init(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: attr is initialized + let ret = unsafe { + libc::pthread_mutexattr_settype(attr, libc::PTHREAD_MUTEX_NORMAL) + }; + if ret != 0 { + // SAFETY: attr is initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: slot is valid + unsafe { slot.write(libc::PTHREAD_MUTEX_INITIALIZER) }; + // SAFETY: attr and slot are valid ptrs and attr is initialized + let ret = unsafe { libc::pthread_mutex_init(slot, attr) }; + // SAFETY: attr was initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + Ok(()) + }; + // SAFETY: mutex has been initialized + unsafe { pin_init_from_closure(init) } + } + pin_init!(Self { + data: UnsafeCell::new(data), + raw <- init_raw(), + pin: PhantomPinned, + }? Error) + } + + #[allow(dead_code)] + pub fn lock(&self) -> PThreadMutexGuard<'_, T> { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_lock(self.raw.get()) }; + PThreadMutexGuard { mtx: self } + } + } + + pub struct PThreadMutexGuard<'a, T> { + mtx: &'a PThreadMutex, + } + + impl Drop for PThreadMutexGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_unlock(self.mtx.raw.get()) }; + } + } + + impl Deref for PThreadMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } + } + + impl DerefMut for PThreadMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } + } +} + +#[cfg_attr(test, test)] +#[cfg_attr(all(test, miri), ignore)] +fn main() { + #[cfg(all(any(feature = "std", feature = "alloc"), not(windows)))] + { + use core::pin::Pin; + use pin_init::*; + use pthread_mtx::*; + use std::{ + sync::Arc, + thread::{sleep, Builder}, + time::Duration, + }; + let mtx: Pin>> = Arc::try_pin_init(PThreadMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", *mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs new file mode 100644 index 000000000..8dd52313c --- /dev/null +++ b/rust/pin-init/examples/static_init.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(unused_imports)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit, + ops, + pin::Pin, + time::Duration, +}; +use pin_init::*; +#[cfg(feature = "std")] +use std::{ + sync::Arc, + thread::{sleep, Builder}, +}; + +#[allow(unused_attributes)] +mod mutex; +use mutex::*; + +pub struct StaticInit { + cell: UnsafeCell>, + init: Cell>, + lock: SpinLock, + present: Cell, +} + +unsafe impl Sync for StaticInit {} +unsafe impl Send for StaticInit {} + +impl> StaticInit { + pub const fn new(init: I) -> Self { + Self { + cell: UnsafeCell::new(MaybeUninit::uninit()), + init: Cell::new(Some(init)), + lock: SpinLock::new(), + present: Cell::new(false), + } + } +} + +impl> ops::Deref for StaticInit { + type Target = T; + fn deref(&self) -> &Self::Target { + if self.present.get() { + unsafe { (*self.cell.get()).assume_init_ref() } + } else { + println!("acquire spinlock on static init"); + let _guard = self.lock.acquire(); + println!("rechecking present..."); + std::thread::sleep(std::time::Duration::from_millis(200)); + if self.present.get() { + return unsafe { (*self.cell.get()).assume_init_ref() }; + } + println!("doing init"); + let ptr = self.cell.get().cast::(); + match self.init.take() { + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, + None => unsafe { core::hint::unreachable_unchecked() }, + } + self.present.set(true); + unsafe { (*self.cell.get()).assume_init_ref() } + } + } +} + +pub struct CountInit; + +unsafe impl PinInit> for CountInit { + unsafe fn __init(self, slot: *mut CMutex) -> Result<(), core::convert::Infallible> { + let init = CMutex::new(0); + std::thread::sleep(std::time::Duration::from_millis(1000)); + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) + } +} + +pub static COUNT: StaticInit, CountInit> = StaticInit::new(CountInit); + +fn main() { + #[cfg(feature = "std")] + { + let mtx: Pin>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *COUNT.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *COUNT.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}, {:?}", *mtx.lock(), *COUNT.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs new file mode 100644 index 000000000..c7d9b3e62 --- /dev/null +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::fmt::Display; + +use proc_macro2::TokenStream; +use quote::quote_spanned; +use syn::{spanned::Spanned, Error}; + +pub(crate) struct DiagCtxt(TokenStream); +pub(crate) struct ErrorGuaranteed(()); + +impl DiagCtxt { + pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed { + let error = Error::new(span.span(), msg); + self.0.extend(error.into_compile_error()); + ErrorGuaranteed(()) + } + + pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) { + // Have the message start on a new line for visual clarity. + let msg = format!("\n{}", msg); + self.0.extend(quote_spanned!(span.span() => + // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable. + const _: () = { + #[deprecated = #msg] + const fn warn() {} + warn(); + }; + )); + } + + pub(crate) fn with( + fun: impl FnOnce(&mut DiagCtxt) -> Result, + ) -> TokenStream { + let mut dcx = Self(TokenStream::new()); + match fun(&mut dcx) { + Ok(mut stream) => { + stream.extend(dcx.0); + stream + } + Err(ErrorGuaranteed(())) => dcx.0, + } + } +} diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs new file mode 100644 index 000000000..fd0b5ea4a --- /dev/null +++ b/rust/pin-init/internal/src/init.rs @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote}; +use syn::{ + braced, + parse::{End, Parse}, + parse_quote, + punctuated::Punctuated, + spanned::Spanned, + token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Path, Token, Type, +}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) struct Initializer { + attrs: Vec, + this: Option, + path: Path, + brace_token: token::Brace, + fields: Punctuated, + rest: Option<(Token![..], Expr)>, + error: Option<(Token![?], Type)>, +} + +struct This { + _and_token: Token![&], + ident: Ident, + _in_token: Token![in], +} + +struct InitializerField { + attrs: Vec, + kind: InitializerKind, +} + +enum InitializerKind { + Value { + ident: Ident, + value: Option<(Token![:], Expr)>, + }, + Init { + ident: Ident, + _left_arrow_token: Token![<-], + value: Expr, + }, + Code { + _underscore_token: Token![_], + _colon_token: Token![:], + block: Block, + }, +} + +impl InitializerKind { + fn ident(&self) -> Option<&Ident> { + match self { + Self::Value { ident, .. } | Self::Init { ident, .. } => Some(ident), + Self::Code { .. } => None, + } + } +} + +enum InitializerAttribute { + DefaultError(DefaultErrorAttribute), +} + +struct DefaultErrorAttribute { + ty: Box, +} + +pub(crate) fn expand( + Initializer { + attrs, + this, + path, + brace_token, + fields, + rest, + error, + }: Initializer, + default_error: Option<&'static str>, + pinned: bool, + dcx: &mut DiagCtxt, +) -> Result { + let error = error.map_or_else( + || { + if let Some(default_error) = attrs.iter().fold(None, |acc, attr| { + #[expect(irrefutable_let_patterns)] + if let InitializerAttribute::DefaultError(DefaultErrorAttribute { ty }) = attr { + Some(ty.clone()) + } else { + acc + } + }) { + default_error + } else if let Some(default_error) = default_error { + syn::parse_str(default_error).unwrap() + } else { + dcx.error(brace_token.span.close(), "expected `? ` after `}`"); + parse_quote!(::core::convert::Infallible) + } + }, + |(_, err)| Box::new(err), + ); + let slot = format_ident!("slot"); + let (has_data_trait, get_data, init_from_closure) = if pinned { + ( + format_ident!("HasPinData"), + format_ident!("__pin_data"), + format_ident!("pin_init_from_closure"), + ) + } else { + ( + format_ident!("HasInitData"), + format_ident!("__init_data"), + format_ident!("init_from_closure"), + ) + }; + let init_kind = get_init_kind(rest, dcx); + let zeroable_check = match init_kind { + InitKind::Normal => quote!(), + InitKind::Zeroing => quote! { + // The user specified `..Zeroable::zeroed()` at the end of the list of fields. + // Therefore we check if the struct implements `Zeroable` and then zero the memory. + // This allows us to also remove the check that all fields are present (since we + // already set the memory to zero and that is a valid bit pattern). + fn assert_zeroable(_: *mut T) + where T: ::pin_init::Zeroable + {} + // Ensure that the struct is indeed `Zeroable`. + assert_zeroable(#slot); + // SAFETY: The type implements `Zeroable` by the check above. + unsafe { ::core::ptr::write_bytes(#slot, 0, 1) }; + }, + }; + let this = match this { + None => quote!(), + Some(This { ident, .. }) => quote! { + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + let #ident = unsafe { ::core::ptr::NonNull::new_unchecked(slot) }; + }, + }; + // `mixed_site` ensures that the data is not accessible to the user-controlled code. + let data = Ident::new("__data", Span::mixed_site()); + let init_fields = init_fields(&fields, pinned, &data, &slot); + let field_check = make_field_check(&fields, init_kind, &path); + Ok(quote! {{ + // Get the data about fields from the supplied type. + // SAFETY: TODO + let #data = unsafe { + use ::pin_init::__internal::#has_data_trait; + // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit + // generics (which need to be present with that syntax). + #path::#get_data() + }; + // Ensure that `#data` really is of type `#data` and help with type inference: + let init = #data.__make_closure::<_, #error>( + move |slot| { + #zeroable_check + #this + #init_fields + #field_check + // SAFETY: we are the `init!` macro that is allowed to call this. + Ok(unsafe { ::pin_init::__internal::InitOk::new() }) + } + ); + let init = move |slot| -> ::core::result::Result<(), #error> { + init(slot).map(|__InitOk| ()) + }; + // SAFETY: TODO + unsafe { ::pin_init::#init_from_closure::<_, #error>(init) } + }}) +} + +enum InitKind { + Normal, + Zeroing, +} + +fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKind { + let Some((dotdot, expr)) = rest else { + return InitKind::Normal; + }; + match &expr { + Expr::Call(ExprCall { func, args, .. }) if args.is_empty() => match &**func { + Expr::Path(ExprPath { + attrs, + qself: None, + path: + Path { + leading_colon: None, + segments, + }, + }) if attrs.is_empty() + && segments.len() == 2 + && segments[0].ident == "Zeroable" + && segments[0].arguments.is_none() + && segments[1].ident == "init_zeroed" + && segments[1].arguments.is_none() => + { + return InitKind::Zeroing; + } + _ => {} + }, + _ => {} + } + dcx.error( + dotdot.span().join(expr.span()).unwrap_or(expr.span()), + "expected nothing or `..Zeroable::init_zeroed()`.", + ); + InitKind::Normal +} + +/// Generate the code that initializes the fields of the struct using the initializers in `field`. +fn init_fields( + fields: &Punctuated, + pinned: bool, + data: &Ident, + slot: &Ident, +) -> TokenStream { + let mut guards = vec![]; + let mut guard_attrs = vec![]; + let mut res = TokenStream::new(); + for InitializerField { attrs, kind } in fields { + let cfgs = { + let mut cfgs = attrs.clone(); + cfgs.retain(|attr| attr.path().is_ident("cfg")); + cfgs + }; + + let ident = match kind { + InitializerKind::Value { ident, .. } => ident, + InitializerKind::Init { ident, .. } => ident, + InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; + res.extend(quote! { + #(#attrs)* + { + #(#stmt)* + } + }); + continue; + } + }; + + let slot = if pinned { + quote! { + // SAFETY: + // - `slot` is valid and properly aligned. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { #data.#ident(#slot) }) + } + } else { + quote! { + // For `init!()` macro, everything is unpinned. + // SAFETY: + // - `&raw mut (*slot).#ident` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { + ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( + &raw mut (*#slot).#ident + ) + }) + } + }; + + // `mixed_site` ensures that the guard is not accessible to the user-controlled code. + let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + + let init = match kind { + InitializerKind::Value { ident, value } => { + let value = value + .as_ref() + .map(|(_, value)| quote!(#value)) + .unwrap_or_else(|| quote!(#ident)); + + quote! { + #(#attrs)* + let mut #guard = #slot.write(#value); + + } + } + InitializerKind::Init { value, .. } => { + quote! { + #(#attrs)* + let mut #guard = #slot.init(#value)?; + } + } + InitializerKind::Code { .. } => unreachable!(), + }; + + res.extend(quote! { + #init + + #(#cfgs)* + // Allow `non_snake_case` since the same warning is going to be reported for the struct + // field. + #[allow(unused_variables, non_snake_case)] + let #ident = #guard.let_binding(); + }); + + guards.push(guard); + guard_attrs.push(cfgs); + } + quote! { + #res + // If execution reaches this point, all fields have been initialized. Therefore we can now + // dismiss the guards by forgetting them. + #( + #(#guard_attrs)* + ::core::mem::forget(#guards); + )* + } +} + +/// Generate the check for ensuring that every field has been initialized and aligned. +fn make_field_check( + fields: &Punctuated, + init_kind: InitKind, + path: &Path, +) -> TokenStream { + let field_attrs: Vec<_> = fields + .iter() + .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let zeroing_trailer = match init_kind { + InitKind::Normal => None, + InitKind::Zeroing => Some(quote! { + ..::core::mem::zeroed() + }), + }; + quote! { + #[allow(unreachable_code)] + // We use unreachable code to perform field checks. They're still checked by the compiler. + // SAFETY: this code is never executed. + let _ = || unsafe { + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #( + #(#field_attrs)* + let _ = &(*slot).#field_name; + )* + + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #( + #(#field_attrs)* + #field_name: loop {}, + )* + #zeroing_trailer + }) + }; + } +} + +impl Parse for Initializer { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let attrs = input.call(Attribute::parse_outer)?; + let this = input.peek(Token![&]).then(|| input.parse()).transpose()?; + let path = input.parse()?; + let content; + let brace_token = braced!(content in input); + let mut fields = Punctuated::new(); + loop { + let lh = content.lookahead1(); + if lh.peek(End) || lh.peek(Token![..]) { + break; + } else if lh.peek(Ident) || lh.peek(Token![_]) || lh.peek(Token![#]) { + fields.push_value(content.parse()?); + let lh = content.lookahead1(); + if lh.peek(End) { + break; + } else if lh.peek(Token![,]) { + fields.push_punct(content.parse()?); + } else { + return Err(lh.error()); + } + } else { + return Err(lh.error()); + } + } + let rest = content + .peek(Token![..]) + .then(|| Ok::<_, syn::Error>((content.parse()?, content.parse()?))) + .transpose()?; + let error = input + .peek(Token![?]) + .then(|| Ok::<_, syn::Error>((input.parse()?, input.parse()?))) + .transpose()?; + let attrs = attrs + .into_iter() + .map(|a| { + if a.path().is_ident("default_error") { + a.parse_args::() + .map(InitializerAttribute::DefaultError) + } else { + Err(syn::Error::new_spanned(a, "unknown initializer attribute")) + } + }) + .collect::, _>>()?; + Ok(Self { + attrs, + this, + path, + brace_token, + fields, + rest, + error, + }) + } +} + +impl Parse for DefaultErrorAttribute { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + Ok(Self { ty: input.parse()? }) + } +} + +impl Parse for This { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + Ok(Self { + _and_token: input.parse()?, + ident: input.parse()?, + _in_token: input.parse()?, + }) + } +} + +impl Parse for InitializerField { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let attrs = input.call(Attribute::parse_outer)?; + Ok(Self { + attrs, + kind: input.parse()?, + }) + } +} + +impl Parse for InitializerKind { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let lh = input.lookahead1(); + if lh.peek(Token![_]) { + Ok(Self::Code { + _underscore_token: input.parse()?, + _colon_token: input.parse()?, + block: input.parse()?, + }) + } else if lh.peek(Ident) { + let ident = input.parse()?; + let lh = input.lookahead1(); + if lh.peek(Token![<-]) { + Ok(Self::Init { + ident, + _left_arrow_token: input.parse()?, + value: input.parse()?, + }) + } else if lh.peek(Token![:]) { + Ok(Self::Value { + ident, + value: Some((input.parse()?, input.parse()?)), + }) + } else if lh.peek(Token![,]) || lh.peek(End) { + Ok(Self::Value { ident, value: None }) + } else { + Err(lh.error()) + } + } else { + Err(lh.error()) + } + } +} diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs new file mode 100644 index 000000000..60d5093f3 --- /dev/null +++ b/rust/pin-init/internal/src/lib.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! `pin-init` proc macros. + +// Documentation is done in the pin-init crate instead. +#![allow(missing_docs)] + +use proc_macro::TokenStream; +use syn::parse_macro_input; + +use crate::diagnostics::DiagCtxt; + +mod diagnostics; +mod init; +mod pin_data; +mod pinned_drop; +mod zeroable; + +#[proc_macro_attribute] +pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args); + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| pin_data::pin_data(args, input, dcx)).into() +} + +#[proc_macro_attribute] +pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args); + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into() +} + +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| zeroable::derive(input, dcx)).into() +} + +#[proc_macro_derive(MaybeZeroable)] +pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| zeroable::maybe_derive(input, dcx)).into() +} +#[proc_macro] +pub fn init(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), false, dcx)) + .into() +} + +#[proc_macro] +pub fn pin_init(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), true, dcx)).into() +} diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs new file mode 100644 index 000000000..ff194d275 --- /dev/null +++ b/rust/pin-init/internal/src/pin_data.rs @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::{format_ident, quote, ToTokens}; +use syn::{ + parse::{End, Nothing, Parse}, + parse_quote, parse_quote_spanned, + spanned::Spanned, + visit_mut::VisitMut, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, +}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) mod kw { + syn::custom_keyword!(PinnedDrop); +} + +pub(crate) enum Args { + Nothing(Nothing), + #[allow(dead_code)] + PinnedDrop(kw::PinnedDrop), +} + +impl Parse for Args { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let lh = input.lookahead1(); + if lh.peek(End) { + input.parse().map(Self::Nothing) + } else if lh.peek(kw::PinnedDrop) { + input.parse().map(Self::PinnedDrop) + } else { + Err(lh.error()) + } + } +} + +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + +struct FieldInfo<'a> { + field: &'a Field, + pinned: bool, +} + +pub(crate) fn pin_data( + args: Args, + input: Item, + dcx: &mut DiagCtxt, +) -> Result { + let mut struct_ = match input { + Item::Struct(struct_) => struct_, + Item::Enum(enum_) => { + return Err(dcx.error( + enum_.enum_token, + "`#[pin_data]` only supports structs for now", + )); + } + Item::Union(union) => { + return Err(dcx.error( + union.union_token, + "`#[pin_data]` only supports structs for now", + )); + } + rest => { + return Err(dcx.error( + rest, + "`#[pin_data]` can only be applied to struct, enum and union definitions", + )); + } + }; + + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + + // The generics might contain the `Self` type. Since this macro will define a new type with the + // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed + // to this struct definition. Therefore we have to replace `Self` with the concrete name. + let mut replacer = { + let name = &struct_.ident; + let (_, ty_generics, _) = struct_.generics.split_for_impl(); + SelfReplacer(parse_quote!(#name #ty_generics)) + }; + replacer.visit_generics_mut(&mut struct_.generics); + replacer.visit_fields_mut(&mut struct_.fields); + + let fields: Vec> = struct_ + .fields + .iter_mut() + .map(|field| { + let len = field.attrs.len(); + field.attrs.retain(|a| !a.path().is_ident("pin")); + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } + + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); + + FieldInfo { + field: &*field, + pinned: pinned_count != 0, + } + }) + .collect(); + + for field in &fields { + let ident = field.field.ident.as_ref().unwrap(); + + if !field.pinned && is_phantom_pinned(&field.field.ty) { + dcx.warn( + field.field, + format!( + "The field `{ident}` of type `PhantomPinned` only has an effect \ + if it has the `#[pin]` attribute", + ), + ); + } + } + + let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields); + let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args); + let projections = + generate_projections(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + let the_pin_data = + generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields); + + Ok(quote! { + #struct_ + #projections + // We put the rest into this const item, because it then will not be accessible to anything + // outside. + const _: () = { + #the_pin_data + #unpin_impl + #drop_impl + }; + }) +} + +fn is_phantom_pinned(ty: &Type) -> bool { + match ty { + Type::Path(TypePath { qself: None, path }) => { + // Cannot possibly refer to `PhantomPinned` (except alias, but that's on the user). + if path.segments.len() > 3 { + return false; + } + // If there is a `::`, then the path needs to be `::core::marker::PhantomPinned` or + // `::std::marker::PhantomPinned`. + if path.leading_colon.is_some() && path.segments.len() != 3 { + return false; + } + let expected: Vec<&[&str]> = vec![&["PhantomPinned"], &["marker"], &["core", "std"]]; + for (actual, expected) in path.segments.iter().rev().zip(expected) { + if !actual.arguments.is_empty() || expected.iter().all(|e| actual.ident != e) { + return false; + } + } + true + } + _ => false, + } +} + +fn generate_unpin_impl( + ident: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (_, ty_generics, _) = generics.split_for_impl(); + let mut generics_with_pin_lt = generics.clone(); + generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); + generics_with_pin_lt.make_where_clause(); + let ( + impl_generics_with_pin_lt, + ty_generics_with_pin_lt, + Some(WhereClause { + where_token, + predicates, + }), + ) = generics_with_pin_lt.split_for_impl() + else { + unreachable!() + }; + let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { + let ident = f.field.ident.as_ref().unwrap(); + let ty = &f.field.ty; + quote!( + #ident: #ty + ) + }); + quote! { + // This struct will be used for the unpin analysis. It is needed, because only structurally + // pinned fields are relevant whether the struct should implement `Unpin`. + #[allow( + dead_code, // The fields below are never used. + non_snake_case // The warning will be emitted on the struct definition. + )] + struct __Unpin #generics_with_pin_lt + #where_token + #predicates + { + __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>, + __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>, + #(#pinned_fields),* + } + + #[doc(hidden)] + impl #impl_generics_with_pin_lt ::core::marker::Unpin for #ident #ty_generics + #where_token + __Unpin #ty_generics_with_pin_lt: ::core::marker::Unpin, + #predicates + {} + } +} + +fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenStream { + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + let has_pinned_drop = matches!(args, Args::PinnedDrop(_)); + // We need to disallow normal `Drop` implementation, the exact behavior depends on whether + // `PinnedDrop` was specified in `args`. + if has_pinned_drop { + // When `PinnedDrop` was specified we just implement `Drop` and delegate. + quote! { + impl #impl_generics ::core::ops::Drop for #ident #ty_generics + #whr + { + fn drop(&mut self) { + // SAFETY: Since this is a destructor, `self` will not move after this function + // terminates, since it is inaccessible. + let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; + // SAFETY: Since this is a drop function, we can create this token to call the + // pinned destructor of this type. + let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() }; + ::pin_init::PinnedDrop::drop(pinned, token); + } + } + } + } else { + // When no `PinnedDrop` was specified, then we have to prevent implementing drop. + quote! { + // We prevent this by creating a trait that will be implemented for all types implementing + // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, + // if it also implements `Drop` + trait MustNotImplDrop {} + impl MustNotImplDrop for T {} + impl #impl_generics MustNotImplDrop for #ident #ty_generics + #whr + {} + // We also take care to prevent users from writing a useless `PinnedDrop` implementation. + // They might implement `PinnedDrop` correctly for the struct, but forget to give + // `PinnedDrop` as the parameter to `#[pin_data]`. + trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} + impl + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} + impl #impl_generics + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for #ident #ty_generics + #whr + {} + } + } +} + +fn generate_projections( + vis: &Visibility, + ident: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (impl_generics, ty_generics, _) = generics.split_for_impl(); + let mut generics_with_pin_lt = generics.clone(); + generics_with_pin_lt.params.insert(0, parse_quote!('__pin)); + let (_, ty_generics_with_pin_lt, whr) = generics_with_pin_lt.split_for_impl(); + let projection = format_ident!("{ident}Projection"); + let this = format_ident!("this"); + + let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields + .iter() + .map(|field| { + let Field { vis, ident, ty, .. } = &field.field; + + let ident = ident + .as_ref() + .expect("only structs with named fields are supported"); + if field.pinned { + ( + quote!( + #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, + ), + quote!( + // SAFETY: this field is structurally pinned. + #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, + ), + ) + } else { + ( + quote!( + #vis #ident: &'__pin mut #ty, + ), + quote!( + #ident: &mut #this.#ident, + ), + ) + } + }) + .collect(); + let structurally_pinned_fields_docs = fields + .iter() + .filter(|f| f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + let not_structurally_pinned_fields_docs = fields + .iter() + .filter(|f| !f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); + let docs = format!(" Pin-projections of [`{ident}`]"); + quote! { + #[doc = #docs] + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] + #[doc(hidden)] + #vis struct #projection #generics_with_pin_lt + #whr + { + #(#fields_decl)* + ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, + } + + impl #impl_generics #ident #ty_generics + #whr + { + /// Pin-projects all fields of `Self`. + /// + /// These fields are structurally pinned: + #(#[doc = #structurally_pinned_fields_docs])* + /// + /// These fields are **not** structurally pinned: + #(#[doc = #not_structurally_pinned_fields_docs])* + #[inline] + #vis fn project<'__pin>( + self: ::core::pin::Pin<&'__pin mut Self>, + ) -> #projection #ty_generics_with_pin_lt { + // SAFETY: we only give access to `&mut` for fields not structurally pinned. + let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; + #projection { + #(#fields_proj)* + ___pin_phantom_data: ::core::marker::PhantomData, + } + } + } + } +} + +fn generate_the_pin_data( + vis: &Visibility, + struct_name: &Ident, + generics: &Generics, + fields: &[FieldInfo<'_>], +) -> TokenStream { + let (impl_generics, ty_generics, whr) = generics.split_for_impl(); + + // For every field, we create an initializing projection function according to its projection + // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be + // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with + // `Unpinned` which allows initialization via `Init`. + let field_accessors = fields + .iter() + .map(|f| { + let Field { vis, ident, ty, .. } = f.field; + + let field_name = ident + .as_ref() + .expect("only structs with named fields are supported"); + let pin_marker = if f.pinned { + quote!(Pinned) + } else { + quote!(Unpinned) + }; + quote! { + /// # Safety + /// + /// - `slot` is valid and properly aligned. + /// - `(*slot).#field_name` is properly aligned. + /// - `(*slot).#field_name` points to uninitialized and exclusively accessed + /// memory. + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(non_snake_case)] + #[inline(always)] + #vis unsafe fn #field_name( + self, + slot: *mut #struct_name #ty_generics, + ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> { + // SAFETY: + // - If `#pin_marker` is `Pinned`, the corresponding field is structurally + // pinned. + // - Other safety requirements follows the safety requirement. + unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) } + } + } + }) + .collect::(); + quote! { + // We declare this struct which will host all of the projection function for our type. It + // will be invariant over all generic parameters which are inherited from the struct. + #[doc(hidden)] + #vis struct __ThePinData #generics + #whr + { + __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>, + } + + impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics + #whr + { + #[inline] + fn clone(&self) -> Self { *self } + } + + impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics + #whr + {} + + #[allow(dead_code)] // Some functions might never be used and private. + impl #impl_generics __ThePinData #ty_generics + #whr + { + /// Type inference helper function. + #[inline(always)] + #vis fn __make_closure<__F, __E>(self, f: __F) -> __F + where + __F: FnOnce(*mut #struct_name #ty_generics) -> + ::core::result::Result<::pin_init::__internal::InitOk, __E>, + { + f + } + + #field_accessors + } + + // SAFETY: We have added the correct projection functions above to `__ThePinData` and + // we also use the least restrictive generics possible. + unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics + #whr + { + type PinData = __ThePinData #ty_generics; + + #[inline] + unsafe fn __pin_data() -> Self::PinData { + __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } + } + } + } +} + +struct SelfReplacer(PathSegment); + +impl VisitMut for SelfReplacer { + fn visit_path_mut(&mut self, i: &mut syn::Path) { + if i.is_ident("Self") { + let span = i.span(); + let seg = &self.0; + *i = parse_quote_spanned!(span=> #seg); + } else { + syn::visit_mut::visit_path_mut(self, i); + } + } + + fn visit_path_segment_mut(&mut self, seg: &mut PathSegment) { + if seg.ident == "Self" { + let span = seg.span(); + let this = &self.0; + *seg = parse_quote_spanned!(span=> #this); + } else { + syn::visit_mut::visit_path_segment_mut(self, seg); + } + } + + fn visit_item_mut(&mut self, _: &mut Item) { + // Do not descend into items, since items reset/change what `Self` refers to. + } +} diff --git a/rust/pin-init/internal/src/pinned_drop.rs b/rust/pin-init/internal/src/pinned_drop.rs new file mode 100644 index 000000000..a20ac314c --- /dev/null +++ b/rust/pin-init/internal/src/pinned_drop.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse::Nothing, parse_quote, spanned::Spanned, ImplItem, ItemImpl, Token}; + +use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; + +pub(crate) fn pinned_drop( + _args: Nothing, + mut input: ItemImpl, + dcx: &mut DiagCtxt, +) -> Result { + if let Some(unsafety) = input.unsafety { + dcx.error(unsafety, "implementing `PinnedDrop` is safe"); + } + input.unsafety = Some(Token![unsafe](input.impl_token.span)); + match &mut input.trait_ { + Some((not, path, _for)) => { + if let Some(not) = not { + dcx.error(not, "cannot implement `!PinnedDrop`"); + } + for (seg, expected) in path + .segments + .iter() + .rev() + .zip(["PinnedDrop", "pin_init", ""]) + { + if expected.is_empty() || seg.ident != expected { + dcx.error(seg, "bad import path for `PinnedDrop`"); + } + if !seg.arguments.is_none() { + dcx.error(&seg.arguments, "unexpected arguments for `PinnedDrop` path"); + } + } + *path = parse_quote!(::pin_init::PinnedDrop); + } + None => { + let span = input + .impl_token + .span + .join(input.self_ty.span()) + .unwrap_or(input.impl_token.span); + dcx.error( + span, + "expected `impl ... PinnedDrop for ...`, got inherent impl", + ); + } + } + for item in &mut input.items { + if let ImplItem::Fn(fn_item) = item { + if fn_item.sig.ident == "drop" { + fn_item + .sig + .inputs + .push(parse_quote!(_: ::pin_init::__internal::OnlyCallFromDrop)); + } + } + } + Ok(quote!(#input)) +} diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs new file mode 100644 index 000000000..b11feaeb1 --- /dev/null +++ b/rust/pin-init/internal/src/zeroable.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse_quote, Data, DeriveInput, Field, Fields}; + +use crate::{diagnostics::ErrorGuaranteed, DiagCtxt}; + +pub(crate) fn derive( + input: DeriveInput, + dcx: &mut DiagCtxt, +) -> Result { + let fields = match input.data { + Data::Struct(data_struct) => data_struct.fields, + Data::Union(data_union) => Fields::Named(data_union.fields), + Data::Enum(data_enum) => { + return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum")); + } + }; + let name = input.ident; + let mut generics = input.generics; + for param in generics.type_params_mut() { + param.bounds.insert(0, parse_quote!(::pin_init::Zeroable)); + } + let (impl_gen, ty_gen, whr) = generics.split_for_impl(); + let field_type = fields.iter().map(|field| &field.ty); + Ok(quote! { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen + #whr + {} + const _: () = { + fn assert_zeroable() {} + fn ensure_zeroable #impl_gen () + #whr + { + #( + assert_zeroable::<#field_type>(); + )* + } + }; + }) +} + +pub(crate) fn maybe_derive( + input: DeriveInput, + dcx: &mut DiagCtxt, +) -> Result { + let fields = match input.data { + Data::Struct(data_struct) => data_struct.fields, + Data::Union(data_union) => Fields::Named(data_union.fields), + Data::Enum(data_enum) => { + return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum")); + } + }; + let name = input.ident; + let mut generics = input.generics; + for param in generics.type_params_mut() { + param.bounds.insert(0, parse_quote!(::pin_init::Zeroable)); + } + for Field { ty, .. } in fields { + generics + .make_where_clause() + .predicates + // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` + // feature . + .push(parse_quote!(#ty: for<'__dummy> ::pin_init::Zeroable)); + } + let (impl_gen, ty_gen, whr) = generics.split_for_impl(); + Ok(quote! { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen + #whr + {} + }) +} diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs new file mode 100644 index 000000000..8e9fd18b9 --- /dev/null +++ b/rust/pin-init/src/__internal.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module contains library internal items. +//! +//! These items must not be used outside of this crate and the pin-init-internal crate located at +//! `../internal`. + +use super::*; + +/// Zero-sized type used to mark a type as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// See the [nomicon] for what subtyping is. See also [this table]. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html +/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html +/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns +#[repr(transparent)] +pub struct PhantomInvariant(PhantomData T>); + +impl Clone for PhantomInvariant { + #[inline(always)] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for PhantomInvariant {} + +impl Default for PhantomInvariant { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl PhantomInvariant { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +/// Zero-sized type used to mark a lifetime as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html +#[repr(transparent)] +#[derive(Clone, Copy, Default)] +pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); + +impl PhantomInvariantLifetime<'_> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomInvariant::new()) + } +} + +/// Token type to signify successful initialization. +/// +/// Can only be constructed via the unsafe [`Self::new`] function. The initializer macros use this +/// token type to prevent returning `Ok` from an initializer without initializing all fields. +pub struct InitOk(()); + +impl InitOk { + /// Creates a new token. + /// + /// # Safety + /// + /// This function may only be called from the `init!` macro in `../internal/src/init.rs`. + #[inline(always)] + pub unsafe fn new() -> Self { + Self(()) + } +} + +/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate +/// the pin projections within the initializers. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasPinData { + type PinData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __pin_data() -> Self::PinData; +} + +/// This trait is automatically implemented for every type. It aims to provide the same type +/// inference help as `HasPinData`. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasInitData { + type InitData; + + #[expect(clippy::missing_safety_doc)] + unsafe fn __init_data() -> Self::InitData; +} + +pub struct AllData(PhantomInvariant); + +impl Clone for AllData { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for AllData {} + +impl AllData { + /// Type inference helper function. + #[inline(always)] + pub fn __make_closure(self, f: F) -> F + where + F: FnOnce(*mut T) -> Result, + { + f + } +} + +// SAFETY: TODO. +unsafe impl HasInitData for T { + type InitData = AllData; + + #[inline] + unsafe fn __init_data() -> Self::InitData { + AllData(PhantomInvariant::new()) + } +} + +/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. +/// +/// # Invariants +/// +/// If `self.is_init` is true, then `self.value` is initialized. +/// +/// [`stack_pin_init`]: crate::stack_pin_init +pub struct StackInit { + value: MaybeUninit, + is_init: bool, +} + +impl Drop for StackInit { + #[inline] + fn drop(&mut self) { + if self.is_init { + // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is + // true, `self.value` is initialized. + unsafe { self.value.assume_init_drop() }; + } + } +} + +impl StackInit { + /// Creates a new [`StackInit`] that is uninitialized. Use [`stack_pin_init`] instead of this + /// primitive. + /// + /// [`stack_pin_init`]: crate::stack_pin_init + #[inline] + pub fn uninit() -> Self { + Self { + value: MaybeUninit::uninit(), + is_init: false, + } + } + + /// Initializes the contents and returns the result. + #[inline] + pub fn init(self: Pin<&mut Self>, init: impl PinInit) -> Result, E> { + // SAFETY: We never move out of `this`. + let this = unsafe { Pin::into_inner_unchecked(self) }; + // The value is currently initialized, so it needs to be dropped before we can reuse + // the memory (this is a safety guarantee of `Pin`). + if this.is_init { + this.is_init = false; + // SAFETY: `this.is_init` was true and therefore `this.value` is initialized. + unsafe { this.value.assume_init_drop() }; + } + // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. + unsafe { init.__init(this.value.as_mut_ptr())? }; + // INVARIANT: `this.value` is initialized above. + this.is_init = true; + // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. + Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) }) + } +} + +#[test] +#[cfg(feature = "std")] +fn stack_init_reuse() { + use ::std::{borrow::ToOwned, println, string::String}; + use core::pin::pin; + + #[derive(Debug)] + struct Foo { + a: usize, + b: String, + } + let mut slot: Pin<&mut StackInit> = pin!(StackInit::uninit()); + let value: Result, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 42, + b: "Hello".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); + let value: Result, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 24, + b: "world!".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); +} + +// Marker types that determines type of `DropGuard`'s let bindings. +pub struct Pinned; +pub struct Unpinned; + +/// Represent an uninitialized field. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory. +/// - If `P` is `Pinned`, then `ptr` is structurally pinned. +pub struct Slot { + ptr: *mut T, + _phantom: PhantomData

, +} + +impl Slot { + /// # Safety + /// + /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed + /// memory. + /// - If `P` is `Pinned`, then `ptr` is structurally pinned. + #[inline(always)] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + _phantom: PhantomData, + } + } + + /// Initialize the field by value. + #[inline(always)] + pub fn write(self, value: T) -> DropGuard + where + T: Sized, + { + // SAFETY: `self.ptr` is a valid and aligned pointer for write. + unsafe { self.ptr.write(value) } + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + // - If `P` is `Pinned`, `self.ptr` is pinned. + unsafe { DropGuard::new(self.ptr) } + } +} + +impl Slot { + /// Initialize the field. + #[inline(always)] + pub fn init(self, init: impl Init) -> Result, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `slot`; + // also `self` is consumed so it cannot be touched further. + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +impl Slot { + /// Initialize the field. + #[inline(always)] + pub fn init(self, init: impl PinInit) -> Result, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and pinned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +/// When a value of this type is dropped, it drops a `T`. +/// +/// Can be forgotten to prevent the drop. +/// +/// # Invariants +/// +/// - `ptr` is valid and properly aligned. +/// - `*ptr` is initialized and owned by this guard. +/// - if `P` is `Pinned`, `ptr` is pinned. +pub struct DropGuard { + ptr: *mut T, + phantom: PhantomData

, +} + +impl DropGuard { + /// Creates a drop guard and transfer the ownership of the pointer content. + /// + /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`]. + /// + /// # Safety + /// + /// - `ptr` is valid and properly aligned. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. + /// - if `P` is `Pinned`, `ptr` is pinned. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: By safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } +} + +impl DropGuard { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } +} + +impl DropGuard { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&mut T> { + // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and + // pinned per type invariant. + unsafe { Pin::new_unchecked(&mut *self.ptr) } + } +} + +impl Drop for DropGuard { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. + unsafe { ptr::drop_in_place(self.ptr) } + } +} + +/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely +/// created struct. This is needed, because the `drop` function is safe, but should not be called +/// manually. +pub struct OnlyCallFromDrop(()); + +impl OnlyCallFromDrop { + /// # Safety + /// + /// This function should only be called from the [`Drop::drop`] function and only be used to + /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type. + pub unsafe fn new() -> Self { + Self(()) + } +} + +/// Initializer that always fails. +/// +/// Used by [`assert_pinned!`]. +/// +/// [`assert_pinned!`]: crate::assert_pinned +pub struct AlwaysFail { + _t: PhantomData, +} + +impl AlwaysFail { + /// Creates a new initializer that always fails. + #[inline] + pub fn new() -> Self { + Self { _t: PhantomData } + } +} + +impl Default for AlwaysFail { + #[inline] + fn default() -> Self { + Self::new() + } +} + +// SAFETY: `__init` always fails, which is always okay. +unsafe impl PinInit for AlwaysFail { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { + Err(()) + } +} diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs new file mode 100644 index 000000000..471652e86 --- /dev/null +++ b/rust/pin-init/src/alloc.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(all(feature = "alloc", not(feature = "std")))] +use alloc::{boxed::Box, sync::Arc}; +#[cfg(feature = "alloc")] +use core::alloc::AllocError; +use core::{mem::MaybeUninit, pin::Pin}; +#[cfg(feature = "std")] +use std::sync::Arc; + +#[cfg(not(feature = "alloc"))] +type AllocError = core::convert::Infallible; + +use crate::{ + init_from_closure, pin_init_from_closure, InPlaceWrite, Init, PinInit, ZeroableOption, +}; + +pub extern crate alloc; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// ). +unsafe impl ZeroableOption for Box {} + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit: Sized { + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] + fn pin_init(init: impl PinInit) -> Result, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_pin_init(init) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init(init: impl Init) -> Result + where + E: From; + + /// Use the given initializer to in-place initialize a `T`. + #[inline] + fn init(init: impl Init) -> Result { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_init(init) + } +} + +#[cfg(feature = "alloc")] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::try_new_uninit()? + }; +} +#[cfg(all(feature = "std", not(feature = "alloc")))] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::new_uninit() + }; +} + +impl InPlaceInit for Box { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + try_new_uninit!(Box).write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + try_new_uninit!(Box).write_init(init) + } +} + +impl InPlaceInit for Arc { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized and this is the only `Arc` to that data. + Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} + +impl InPlaceWrite for Box> { + type Initialized = Box; + + #[inline] + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + #[inline] + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs new file mode 100644 index 000000000..f1463be94 --- /dev/null +++ b/rust/pin-init/src/lib.rs @@ -0,0 +1,1797 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Library to safely and fallibly initialize pinned `struct`s using in-place constructors. +//! +//! [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack +//! overflow. +//! +//! This library's main use-case is in [Rust-for-Linux]. Although this version can be used +//! standalone. +//! +//! There are cases when you want to in-place initialize a struct. For example when it is very big +//! and moving it from the stack is not an option, because it is bigger than the stack itself. +//! Another reason would be that you need the address of the object to initialize it. This stands +//! in direct conflict with Rust's normal process of first initializing an object and then moving +//! it into it's final memory location. For more information, see +//! . +//! +//! This library allows you to do in-place initialization safely. +//! +//! ## Nightly Needed for `alloc` feature +//! +//! This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +//! enabled and thus this feature can only be used with a nightly compiler. When enabling the +//! `alloc` feature, the user will be required to activate `allocator_api` as well. +//! +//! [`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html +//! +//! The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +//! However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +//! will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +//! mode. +//! +//! ## Nightly needed for `unsafe-pinned` feature +//! +//! This feature enables the `Wrapper` implementation on the unstable `core::pin::UnsafePinned` type. +//! This requires the [`unsafe_pinned` unstable feature](https://github.com/rust-lang/rust/issues/125735) +//! and therefore a nightly compiler. Note that this feature is not enabled by default. +//! +//! # Overview +//! +//! To initialize a `struct` with an in-place constructor you will need two things: +//! - an in-place constructor, +//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], +//! [`Box`] or any other smart pointer that supports this library). +//! +//! To get an in-place constructor there are generally three options: +//! - directly creating an in-place constructor using the [`pin_init!`] macro, +//! - a custom function/macro returning an in-place constructor provided by someone else, +//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! +//! Aside from pinned initialization, this library also supports in-place construction without +//! pinning, the macros/types/functions are generally named like the pinned variants without the +//! `pin_` prefix. +//! +//! # Examples +//! +//! Throughout the examples we will often make use of the `CMutex` type which can be found in +//! `../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +//! the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +//! requires it to be pinned to be locked and thus is a prime candidate for using this library. +//! +//! ## Using the [`pin_init!`] macro +//! +//! If you want to use [`PinInit`], then you will have to annotate your `struct` with +//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via +//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::pin::Pin; +//! use pin_init::{pin_data, pin_init, InPlaceInit}; +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! a: CMutex, +//! b: u32, +//! } +//! +//! let foo = pin_init!(Foo { +//! a <- CMutex::new(42), +//! b: 24, +//! }); +//! # let _ = Box::pin_init(foo); +//! ``` +//! +//! `foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like +//! (or just the stack) to actually initialize a `Foo`: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::{alloc::AllocError, pin::Pin}; +//! # use pin_init::*; +//! # +//! # #[pin_data] +//! # struct Foo { +//! # #[pin] +//! # a: CMutex, +//! # b: u32, +//! # } +//! # +//! # let foo = pin_init!(Foo { +//! # a <- CMutex::new(42), +//! # b: 24, +//! # }); +//! let foo: Result>, AllocError> = Box::pin_init(foo); +//! ``` +//! +//! For more information see the [`pin_init!`] macro. +//! +//! ## Using a custom function/macro that returns an initializer +//! +//! Many types that use this library supply a function/macro that returns an initializer, because +//! the above method only works for types where you can access the fields. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use pin_init::*; +//! # use std::sync::Arc; +//! # use core::pin::Pin; +//! let mtx: Result>>, _> = Arc::pin_init(CMutex::new(42)); +//! ``` +//! +//! To declare an init macro/function you just return an [`impl PinInit`]: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # use pin_init::*; +//! # #[path = "../examples/error.rs"] mod error; use error::Error; +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! #[pin_data] +//! struct DriverData { +//! #[pin] +//! status: CMutex, +//! buffer: Box<[u8; 1_000_000]>, +//! } +//! +//! impl DriverData { +//! fn new() -> impl PinInit { +//! pin_init!(Self { +//! status <- CMutex::new(0), +//! buffer: Box::init(pin_init::init_zeroed())?, +//! }? Error) +//! } +//! } +//! ``` +//! +//! ## Manual creation of an initializer +//! +//! Often when working with primitives the previous approaches are not sufficient. That is where +//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +//! [`impl PinInit`] directly from a closure. Of course you have to ensure that the closure +//! actually does the initialization in the correct way. Here are the things to look out for +//! (we are calling the parameter to the closure `slot`): +//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so +//! `slot` now contains a valid bit pattern for the type `T`, +//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so +//! you need to take care to clean up anything if your initialization fails mid-way, +//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of +//! `slot` gets called. +//! +//! ```rust +//! # #![feature(extern_types)] +//! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +//! use core::{ +//! marker::PhantomPinned, +//! cell::UnsafeCell, +//! pin::Pin, +//! mem::MaybeUninit, +//! }; +//! mod bindings { +//! #[repr(C)] +//! pub struct foo { +//! /* fields from C ... */ +//! } +//! extern "C" { +//! pub fn init_foo(ptr: *mut foo); +//! pub fn destroy_foo(ptr: *mut foo); +//! #[must_use = "you must check the error return code"] +//! pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; +//! } +//! } +//! +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! _p: PhantomPinned, +//! #[pin] +//! foo: UnsafeCell>, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = &raw mut (*slot).foo; +//! let foo = UnsafeCell::raw_get(foo).cast::(); +//! +//! // Initialize the `foo` +//! bindings::init_foo(foo); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(foo, flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(foo); +//! Err(err) +//! } else { +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! } +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get().cast::()) }; +//! } +//! } +//! ``` +//! +//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +//! the `kernel` crate. The [`sync`] module is a good starting point. +//! +//! [`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [structurally pinned fields]: +//! https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning +//! [stack]: crate::stack_pin_init +#![cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#![cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#![cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#![cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +//! [`impl PinInit`]: crate::PinInit +//! [`impl PinInit`]: crate::PinInit +//! [`impl Init`]: crate::Init +//! [Rust-for-Linux]: https://rust-for-linux.com/ + +#![forbid(missing_docs, unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![cfg_attr( + all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED), + feature(unsafe_pinned) +)] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), allow(internal_features))] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), feature(rustdoc_internals))] + +use core::{ + cell::UnsafeCell, + convert::Infallible, + marker::PhantomData, + mem::MaybeUninit, + num::*, + pin::Pin, + ptr::{self, NonNull}, +}; + +// This is used by doc-tests -- the proc-macros expand to `::pin_init::...` and without this the +// doc-tests wouldn't have an extern crate named `pin_init`. +#[allow(unused_extern_crates)] +extern crate self as pin_init; + +#[doc(hidden)] +pub mod __internal; + +#[cfg(any(feature = "std", feature = "alloc"))] +mod alloc; +#[cfg(any(feature = "std", feature = "alloc"))] +pub use alloc::InPlaceInit; + +/// Used to specify the pinning information of the fields of a struct. +/// +/// This is somewhat similar in purpose as +/// [pin-project-lite](https://crates.io/crates/pin-project-lite). +/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each +/// field you want to structurally pin. +/// +/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, +/// then `#[pin]` directs the type of initializer that is required. +/// +/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this +/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with +/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// use pin_init::pin_data; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// } +/// ``` +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pin_data; + +/// Used to implement `PinnedDrop` safely. +/// +/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pinned_drop; + +/// Derives the [`Zeroable`] trait for the given `struct` or `union`. +/// +/// This can only be used for `struct`s/`union`s where every field implements the [`Zeroable`] +/// trait. +/// +/// # Examples +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub union SignCast { +/// signed: i64, +/// unsigned: u64, +/// } +/// ``` +pub use ::pin_init_internal::Zeroable; + +/// Derives the [`Zeroable`] trait for the given `struct` or `union` if all fields implement +/// [`Zeroable`]. +/// +/// Contrary to the derive macro named [`macro@Zeroable`], this one silently fails when a field +/// doesn't implement [`Zeroable`]. +/// +/// # Examples +/// +/// ``` +/// use pin_init::MaybeZeroable; +/// +/// // implements `Zeroable` +/// #[derive(MaybeZeroable)] +/// pub struct DriverData { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// +/// // does not implement `Zeroable` +/// #[derive(MaybeZeroable)] +/// pub struct DriverData2 { +/// pub(crate) id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// // this field doesn't implement `Zeroable` +/// other_data: &'static i32, +/// } +/// ``` +pub use ::pin_init_internal::MaybeZeroable; + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_pin_init!(let foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Bar { +/// x: 64, +/// }, +/// })); +/// let foo: Pin<&mut Foo> = foo; +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error +/// type, then use [`stack_try_pin_init!`]. +#[macro_export] +macro_rules! stack_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let Ok(mut $var) = $crate::__internal::StackInit::init($var, val); + }; +} + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// let foo = foo.unwrap(); +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo =? pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// println!("a: {}", &*foo.a.lock()); +/// # Ok::<_, Error>(()) +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the +/// `=` will propagate this error. +#[macro_export] +macro_rules! stack_try_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val); + }; + (let $var:ident $(: $t:ty)? =? $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val)?; + }; +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// The error type defaults to [`Infallible`]; if you need a different one, write `? Error` at the +/// end, after the struct initializer. +/// +/// The syntax is almost identical to that of a normal `struct` initializer: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// a: usize, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit { +/// let a = 42; +/// +/// let initializer = pin_init!(Foo { +/// a, +/// b: Bar { +/// x: 64, +/// }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// +/// Arbitrary Rust expressions can be used to set the value of a variable. +/// +/// The fields are initialized in the order that they appear in the initializer. So it is possible +/// to read already initialized fields using raw pointers. +/// +/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the +/// initializer. +/// +/// # Init-functions +/// +/// When working with this library it is often desired to let others construct your types without +/// giving access to all fields. This is where you would normally write a plain function `new` that +/// would return a new instance of your type. With this library that is also possible. However, +/// there are a few extra things to keep in mind. +/// +/// To create an initializer function, simply declare it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// impl Foo { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// a: 42, +/// b: Bar { +/// x: 64, +/// }, +/// }) +/// } +/// } +/// ``` +/// +/// Users of `Foo` can now create it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// let foo = Box::pin_init(Foo::new()); +/// ``` +/// +/// They can also easily embed it into their own `struct`s: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// #[pin_data] +/// struct FooContainer { +/// #[pin] +/// foo1: Foo, +/// #[pin] +/// foo2: Foo, +/// other: u32, +/// } +/// +/// impl FooContainer { +/// fn new(other: u32) -> impl PinInit { +/// pin_init!(Self { +/// foo1 <- Foo::new(), +/// foo2 <- Foo::new(), +/// other, +/// }) +/// } +/// } +/// ``` +/// +/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. +/// This signifies that the given field is initialized in-place. As with `struct` initializers, just +/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. +/// +/// # Syntax +/// +/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with +/// the following modifications is expected: +/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in +/// order to run arbitrary code. +/// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] +/// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::init_zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. +/// +/// For instance: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::marker::PhantomPinned; +/// #[pin_data] +/// #[derive(Zeroable)] +/// struct Buf { +/// // `ptr` points into `buf`. +/// ptr: *mut u8, +/// buf: [u8; 64], +/// #[pin] +/// pin: PhantomPinned, +/// } +/// +/// let init = pin_init!(&this in Buf { +/// buf: [0; 64], +/// // SAFETY: TODO. +/// ptr: unsafe { (&raw mut (*this.as_ptr()).buf).cast() }, +/// pin: PhantomPinned, +/// }); +/// let init = pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::init_zeroed() +/// }); +/// ``` +/// +/// [`NonNull`]: core::ptr::NonNull +pub use pin_init_internal::pin_init; + +/// Construct an in-place, fallible initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]; if you need a different one, write `? Error` +/// at the end, after the struct initializer. +/// +/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// This initializer is for initializing data in-place that might later be moved. If you want to +/// pin-initialize, use [`pin_init!`]. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::InPlaceInit; +/// use pin_init::{init, Init, init_zeroed}; +/// +/// struct BigBuf { +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init { +/// init!(Self { +/// small <- init_zeroed(), +/// }) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +pub use pin_init_internal::init; + +/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is +/// structurally pinned. +/// +/// # Examples +/// +/// This will succeed: +/// ``` +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// #[pin] +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// This will fail: +/// ```compile_fail +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To +/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can +/// only be used when the macro is invoked from a function body. +/// ``` +/// # use core::pin::Pin; +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// elem: T, +/// } +/// +/// impl Foo { +/// fn project_this(self: Pin<&mut Self>) -> Pin<&mut T> { +/// assert_pinned!(Foo, elem, T, inline); +/// +/// // SAFETY: The field is structurally pinned. +/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! assert_pinned { + ($ty:ty, $field:ident, $field_ty:ty, inline) => { + // SAFETY: This code is unreachable. + let _ = move |ptr: *mut $ty| unsafe { + let data = <$ty as $crate::__internal::HasPinData>::__pin_data(); + _ = data + .$field(ptr) + .init($crate::__internal::AlwaysFail::<$field_ty>::new()); + }; + }; + + ($ty:ty, $field:ident, $field_ty:ty) => { + const _: () = { + $crate::assert_pinned!($ty, $field, $field_ty, inline); + }; + }; +} + +/// A pin-initializer for the type `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`] or even the stack (see [`stack_pin_init!`]). +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. +/// +/// The [`PinInit::__init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +#[cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait PinInit: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + + /// Initializes `slot`. + /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + /// - `slot` will not move until it is dropped, i.e. it will be pinned. + /// If `Self: Init`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![feature(allocator_api)] + /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; + /// # use pin_init::*; + /// let mtx_init = CMutex::new(42); + /// // Make the initializer print the value. + /// let mtx_init = mtx_init.pin_chain(|mtx| { + /// println!("{:?}", mtx.get_data_mut()); + /// Ok(()) + /// }); + /// ``` + #[inline] + fn pin_chain(self, f: F) -> ChainPinInit + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, __internal::PhantomInvariant::new()) + } +} + +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init(slot: *mut T, init: impl PinInit) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl PinInit for ChainPinInit +where + I: PinInit, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) + } +} + +/// An initializer for `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`] or even the stack (see [`stack_pin_init!`]). Because +/// [`PinInit`] is a super trait, you can use every function that takes it as well. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. +/// +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. +/// +#[cfg_attr( + kernel, + doc = "[`Arc`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait Init: PinInit { + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// use pin_init::{init, init_zeroed, Init}; + /// + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// println!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- init_zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + #[inline] + fn chain(self, f: F) -> ChainInit + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, __internal::PhantomInvariant::new()) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function does not rely on the pinning requirement. +unsafe impl Init for ChainInit +where + I: Init, + F: FnOnce(&mut T) -> Result<(), E>, +{ +} + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl PinInit for ChainInit +where + I: Init, + F: FnOnce(&mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) + } +} + +/// Implement `PinInit` and `Init` for closures. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__init` functions. +struct InitClosure(F, __internal::PhantomInvariant); + +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl Init for InitClosure where + F: FnOnce(*mut T) -> Result<(), E> +{ +} + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl PinInit for InitClosure +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + +/// Creates a new [`PinInit`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - may assume that the `slot` does not move if `T: !Unpin`, +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn pin_init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl PinInit { + InitClosure(f, __internal::PhantomInvariant::new()) +} + +/// Creates a new [`Init`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - the `slot` may move after initialization. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl Init { + InitClosure(f, __internal::PhantomInvariant::new()) +} + +/// Changes the to be initialized type. +/// +/// # Safety +/// +/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a +/// pointer must result in a valid `U`. +#[inline] +pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { + // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety + // requirements. + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) } +} + +/// Changes the to be initialized type. +/// +/// # Safety +/// +/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a +/// pointer must result in a valid `U`. +#[inline] +pub const unsafe fn cast_init(init: impl Init) -> impl Init { + // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety + // requirements. + unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) } +} + +/// An initializer that leaves the memory uninitialized. +/// +/// The initializer is a no-op. The `slot` memory is not changed. +#[inline] +pub fn uninit() -> impl Init, E> { + // SAFETY: The memory is allowed to be uninitialized. + unsafe { init_from_closure(|_| Ok(())) } +} + +/// Array initializer from element initializer. +struct ArrayInit(F, __internal::PhantomInvariant); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl PinInit<[T; N], E> for ArrayInit +where + F: FnMut(usize) -> I, + I: PinInit, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl Drop for ArrayInitGuard { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl Init<[T; N], E> for ArrayInit +where + F: FnMut(usize) -> I, + I: Init, +{ +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # use pin_init::*; +/// use pin_init::init_array_from_fn; +/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +#[inline] +pub fn init_array_from_fn( + make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init, +{ + ArrayInit(make_init, __internal::PhantomInvariant::new()) +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// use pin_init::pin_init_array_from_fn; +/// use std::sync::Arc; +/// let array: Pin; 1_000]>> = +/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +#[inline] +pub fn pin_init_array_from_fn( + make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit, +{ + ArrayInit(make_init, __internal::PhantomInvariant::new()) +} + +/// Construct an initializer in a closure and run it. +/// +/// Returns an initializer that first runs the closure and then the initializer returned by it. +/// +/// See also [`init_scope`]. +/// +/// # Examples +/// +/// ``` +/// # use pin_init::*; +/// # #[pin_data] +/// # struct Foo { a: u64, b: isize } +/// # struct Bar { a: u32, b: isize } +/// # fn lookup_bar() -> Result { todo!() } +/// # struct Error; +/// fn init_foo() -> impl PinInit { +/// pin_init_scope(|| { +/// let bar = lookup_bar()?; +/// Ok(pin_init!(Foo { a: bar.a.into(), b: bar.b }? Error)) +/// }) +/// } +/// ``` +/// +/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the +/// initializer itself will fail with that error. If it returned `Ok`, then it will run the +/// initializer returned by the [`pin_init!`] invocation. +#[inline] +pub fn pin_init_scope(make_init: F) -> impl PinInit +where + F: FnOnce() -> Result, + I: PinInit, +{ + // SAFETY: + // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. + unsafe { + pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { + let init = make_init()?; + init.__init(slot) + }) + } +} + +/// Construct an initializer in a closure and run it. +/// +/// Returns an initializer that first runs the closure and then the initializer returned by it. +/// +/// See also [`pin_init_scope`]. +/// +/// # Examples +/// +/// ``` +/// # use pin_init::*; +/// # struct Foo { a: u64, b: isize } +/// # struct Bar { a: u32, b: isize } +/// # fn lookup_bar() -> Result { todo!() } +/// # struct Error; +/// fn init_foo() -> impl Init { +/// init_scope(|| { +/// let bar = lookup_bar()?; +/// Ok(init!(Foo { a: bar.a.into(), b: bar.b }? Error)) +/// }) +/// } +/// ``` +/// +/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the +/// initializer itself will fail with that error. If it returned `Ok`, then it will run the +/// initializer returned by the [`init!`] invocation. +#[inline] +pub fn init_scope(make_init: F) -> impl Init +where + F: FnOnce() -> Result, + I: Init, +{ + // SAFETY: + // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. + unsafe { + init_from_closure(move |slot: *mut T| -> Result<(), E> { + let init = make_init()?; + init.__init(slot) + }) + } +} + +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl Init for T {} + +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of +// `slot`. Additionally, all pinning invariants of `T` are upheld. +unsafe impl PinInit for T { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self) }; + Ok(()) + } +} + +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl Init for Result {} + +// SAFETY: when the `__init` function returns with +// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. +// - `Err(err)`, slot was not written to. +unsafe impl PinInit for Result { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self?) }; + Ok(()) + } +} + +/// Smart pointer containing uninitialized memory and that can write a value. +pub trait InPlaceWrite { + /// The type `Self` turns into when the contents are initialized. + type Initialized; + + /// Use the given initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_init(self, init: impl Init) -> Result; + + /// Use the given pin-initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_pin_init(self, init: impl PinInit) -> Result, E>; +} + +impl InPlaceWrite for &'static mut MaybeUninit { + type Initialized = &'static mut T; + + #[inline] + fn write_init(self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + + // SAFETY: `slot` is a valid pointer to uninitialized memory. + unsafe { init.__init(slot)? }; + + // SAFETY: The above call initialized the memory. + unsafe { Ok(self.assume_init_mut()) } + } + + #[inline] + fn write_pin_init(self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + + // SAFETY: `slot` is a valid pointer to uninitialized memory. + // + // The `'static` borrow guarantees the data will not be + // moved/invalidated until it gets dropped (which is never). + unsafe { init.__init(slot)? }; + + // SAFETY: The above call initialized the memory. + Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) + } +} + +/// Trait facilitating pinned destruction. +/// +/// Use [`pinned_drop`] to implement this trait safely: +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// use core::pin::Pin; +/// #[pin_data(PinnedDrop)] +/// struct Foo { +/// #[pin] +/// mtx: CMutex, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for Foo { +/// fn drop(self: Pin<&mut Self>) { +/// println!("Foo is being dropped!"); +/// } +/// } +/// ``` +/// +/// # Safety +/// +/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. +pub unsafe trait PinnedDrop: __internal::HasPinData { + /// Executes the pinned destructor of this type. + /// + /// While this function is marked safe, it is actually unsafe to call it manually. For this + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code + /// and thus prevents this function from being called where it should not. + /// + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute + /// automatically. + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); +} + +/// Marker trait for types that can be initialized by writing just zeroes. +/// +/// # Safety +/// +/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, +/// this is not UB: +/// +/// ```rust,ignore +/// let val: Self = unsafe { core::mem::zeroed() }; +/// ``` +pub unsafe trait Zeroable { + /// Create a new zeroed `Self`. + /// + /// The returned initializer will write `0x00` to every byte of the given `slot`. + #[inline] + fn init_zeroed() -> impl Init + where + Self: Sized, + { + init_zeroed() + } + + /// Create a `Self` consisting of all zeroes. + /// + /// Whenever a type implements [`Zeroable`], this function should be preferred over + /// [`core::mem::zeroed()`] or using `MaybeUninit::zeroed().assume_init()`. + /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// + /// # Examples + /// + /// ``` + /// use pin_init::Zeroable; + /// + /// #[derive(Zeroable)] + /// struct Point { + /// x: u32, + /// y: u32, + /// } + /// + /// let point: Point = Zeroable::zeroed(); + /// assert_eq!(point.x, 0); + /// assert_eq!(point.y, 0); + /// ``` + #[inline] + fn zeroed() -> Self + where + Self: Sized, + { + zeroed() + } +} + +/// Create an initializer for a zeroed `T`. +/// +/// The returned initializer will write `0x00` to every byte of the given `slot`. +#[inline] +pub fn init_zeroed() -> impl Init { + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` + // and because we write all zeroes, the memory is initialized. + unsafe { + init_from_closure(|slot: *mut T| { + slot.write_bytes(0, 1); + Ok(()) + }) + } +} + +/// Create a `T` consisting of all zeroes. +/// +/// Whenever a type implements [`Zeroable`], this function should be preferred over +/// [`core::mem::zeroed()`] or using `MaybeUninit::zeroed().assume_init()`. +/// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// +/// # Examples +/// +/// ``` +/// use pin_init::{Zeroable, zeroed}; +/// +/// #[derive(Zeroable)] +/// struct Point { +/// x: u32, +/// y: u32, +/// } +/// +/// let point: Point = zeroed(); +/// assert_eq!(point.x, 0); +/// assert_eq!(point.y, 0); +/// ``` +#[inline] +pub const fn zeroed() -> T { + // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. + unsafe { core::mem::zeroed() } +} + +macro_rules! impl_zeroable { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? Zeroable for $t {})* + }; +} + +impl_zeroable! { + // SAFETY: All primitives that are allowed to be zero. + bool, + char, + u8, u16, u32, u64, u128, usize, + i8, i16, i32, i64, i128, isize, + f32, f64, + + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // . The Rust Reference + // also has information on undefined behavior: + // . + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {} PhantomData, core::marker::PhantomPinned, (), + + // SAFETY: Type is allowed to take any value, including all zeros. + {} MaybeUninit, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {} UnsafeCell, + + // SAFETY: `null` pointer is valid. + // + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be + // null. + // + // When `Pointee` gets stabilized, we could use + // `T: ?Sized where ::Metadata: Zeroable` + {} *mut T, {} *const T, + + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be + // zero. + {} *mut [T], {} *const [T], *mut str, *const str, + + // SAFETY: `T` is `Zeroable`. + {} [T; N], {} Wrapping, +} + +macro_rules! impl_tuple_zeroable { + ($first:ident, $(,)?) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for tuples up to 10 items long. + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable> Zeroable for ($first,) {} + }; + ($first:ident, $($t:ident),* $(,)?) => { + #[cfg_attr(doc, doc(hidden))] + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} + impl_tuple_zeroable!($($t),* ,); + } +} + +impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); + +/// Marker trait for types that allow `Option` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl Zeroable for Option {} + +macro_rules! impl_fn_zeroable_option { + ([$($abi:literal),* $(,)?] $args:tt) => { + $(impl_fn_zeroable_option!({extern $abi} $args);)* + $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)* + }; + ({$($prefix:tt)*} {$(,)?}) => {}; + ({$($prefix:tt)*} {$ret:ident, $arg:ident $(,)?}) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for function pointers with up to 20 arity. + // SAFETY: function pointers are part of the option layout optimization: + // . + unsafe impl<$ret, $arg> ZeroableOption for $($prefix)* fn($arg) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$arg,}); + }; + ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => { + #[cfg_attr(doc, doc(hidden))] + // SAFETY: function pointers are part of the option layout optimization: + // . + unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$($rest),*,}); + }; +} + +impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U }); + +macro_rules! impl_zeroable_option { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? ZeroableOption for $t {})* + }; +} + +impl_zeroable_option! { + // SAFETY: `Option<&T>` is part of the option layout optimization guarantee: + // . + {} &T, + // SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: + // . + {} &mut T, + // SAFETY: `Option>` is part of the option layout optimization guarantee: + // . + {} NonNull, + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // ). + NonZero, NonZero, NonZero, NonZero, NonZero, NonZero, + NonZero, NonZero, NonZero, NonZero, NonZero, NonZero, +} + +/// This trait allows creating an instance of `Self` which contains exactly one +/// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning). +/// +/// This is useful when using wrapper `struct`s like [`UnsafeCell`] or with new-type `struct`s. +/// +/// # Examples +/// +/// ``` +/// # use core::cell::UnsafeCell; +/// # use pin_init::{pin_data, pin_init, Wrapper}; +/// +/// #[pin_data] +/// struct Foo {} +/// +/// #[pin_data] +/// struct Bar { +/// #[pin] +/// content: UnsafeCell +/// }; +/// +/// let foo_initializer = pin_init!(Foo{}); +/// let initializer = pin_init!(Bar { +/// content <- UnsafeCell::pin_init(foo_initializer) +/// }); +/// ``` +pub trait Wrapper { + /// Creates an pin-initializer for a [`Self`] containing `T` from the `value_init` initializer. + fn pin_init(value_init: impl PinInit) -> impl PinInit; +} + +impl Wrapper for UnsafeCell { + #[inline] + fn pin_init(value_init: impl PinInit) -> impl PinInit { + // SAFETY: `UnsafeCell` has a compatible layout to `T`. + unsafe { cast_pin_init(value_init) } + } +} + +impl Wrapper for MaybeUninit { + #[inline] + fn pin_init(value_init: impl PinInit) -> impl PinInit { + // SAFETY: `MaybeUninit` has a compatible layout to `T`. + unsafe { cast_pin_init(value_init) } + } +} + +#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] +impl Wrapper for core::pin::UnsafePinned { + #[inline] + fn pin_init(init: impl PinInit) -> impl PinInit { + // SAFETY: `UnsafePinned` has a compatible layout to `T`. + unsafe { cast_pin_init(init) } + } +} diff --git a/rust/proc-macro2/README.md b/rust/proc-macro2/README.md new file mode 100644 index 000000000..af044fee4 --- /dev/null +++ b/rust/proc-macro2/README.md @@ -0,0 +1,13 @@ +# `proc-macro2` + +These source files come from the Rust `proc-macro2` crate, version +1.0.101 (released 2025-08-16), hosted in the + repository, licensed under +"Apache-2.0 OR MIT" and only modified to add the SPDX license +identifiers and to remove the `unicode-ident` dependency. + +For copyright details, please see: + + https://github.com/dtolnay/proc-macro2/blob/1.0.101/README.md#license + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-APACHE + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-MIT diff --git a/rust/proc-macro2/detection.rs b/rust/proc-macro2/detection.rs new file mode 100644 index 000000000..3de448cb2 --- /dev/null +++ b/rust/proc-macro2/detection.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use core::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Once; + +static WORKS: AtomicUsize = AtomicUsize::new(0); +static INIT: Once = Once::new(); + +pub(crate) fn inside_proc_macro() -> bool { + match WORKS.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + + INIT.call_once(initialize); + inside_proc_macro() +} + +pub(crate) fn force_fallback() { + WORKS.store(1, Ordering::Relaxed); +} + +pub(crate) fn unforce_fallback() { + initialize(); +} + +#[cfg(not(no_is_available))] +fn initialize() { + let available = proc_macro::is_available(); + WORKS.store(available as usize + 1, Ordering::Relaxed); +} + +// Swap in a null panic hook to avoid printing "thread panicked" to stderr, +// then use catch_unwind to determine whether the compiler's proc_macro is +// working. When proc-macro2 is used from outside of a procedural macro all +// of the proc_macro crate's APIs currently panic. +// +// The Once is to prevent the possibility of this ordering: +// +// thread 1 calls take_hook, gets the user's original hook +// thread 1 calls set_hook with the null hook +// thread 2 calls take_hook, thinks null hook is the original hook +// thread 2 calls set_hook with the null hook +// thread 1 calls set_hook with the actual original hook +// thread 2 calls set_hook with what it thinks is the original hook +// +// in which the user's hook has been lost. +// +// There is still a race condition where a panic in a different thread can +// happen during the interval that the user's original panic hook is +// unregistered such that their hook is incorrectly not called. This is +// sufficiently unlikely and less bad than printing panic messages to stderr +// on correct use of this crate. Maybe there is a libstd feature request +// here. For now, if a user needs to guarantee that this failure mode does +// not occur, they need to call e.g. `proc_macro2::Span::call_site()` from +// the main thread before launching any other threads. +#[cfg(no_is_available)] +fn initialize() { + use std::panic::{self, PanicInfo}; + + type PanicHook = dyn Fn(&PanicInfo) + Sync + Send + 'static; + + let null_hook: Box = Box::new(|_panic_info| { /* ignore */ }); + let sanity_check = &*null_hook as *const PanicHook; + let original_hook = panic::take_hook(); + panic::set_hook(null_hook); + + let works = panic::catch_unwind(proc_macro::Span::call_site).is_ok(); + WORKS.store(works as usize + 1, Ordering::Relaxed); + + let hopefully_null_hook = panic::take_hook(); + panic::set_hook(original_hook); + if sanity_check != &*hopefully_null_hook { + panic!("observed race condition in proc_macro2::inside_proc_macro"); + } +} diff --git a/rust/proc-macro2/extra.rs b/rust/proc-macro2/extra.rs new file mode 100644 index 000000000..55feb5ec7 --- /dev/null +++ b/rust/proc-macro2/extra.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Items which do not have a correspondence to any API in the proc_macro crate, +//! but are necessary to include in proc-macro2. + +use crate::fallback; +use crate::imp; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use crate::Span; +use core::fmt::{self, Debug}; + +/// Invalidate any `proc_macro2::Span` that exist on the current thread. +/// +/// The implementation of `Span` uses thread-local data structures and this +/// function clears them. Calling any method on a `Span` on the current thread +/// created prior to the invalidation will return incorrect values or crash. +/// +/// This function is useful for programs that process more than 232 +/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2 +/// uses 32-bit source locations, and these wrap around when the total source +/// code processed by the same thread exceeds 232 bytes (4 +/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can +/// return wrong data. +/// +/// # Example +/// +/// As of late 2023, there is 200 GB of Rust code published on crates.io. +/// Looking at just the newest version of every crate, it is 16 GB of code. So a +/// workload that involves parsing it all would overflow a 32-bit source +/// location unless spans are being invalidated. +/// +/// ``` +/// use flate2::read::GzDecoder; +/// use std::ffi::OsStr; +/// use std::io::{BufReader, Read}; +/// use std::str::FromStr; +/// use tar::Archive; +/// +/// rayon::scope(|s| { +/// for krate in every_version_of_every_crate() { +/// s.spawn(move |_| { +/// proc_macro2::extra::invalidate_current_thread_spans(); +/// +/// let reader = BufReader::new(krate); +/// let tar = GzDecoder::new(reader); +/// let mut archive = Archive::new(tar); +/// for entry in archive.entries().unwrap() { +/// let mut entry = entry.unwrap(); +/// let path = entry.path().unwrap(); +/// if path.extension() != Some(OsStr::new("rs")) { +/// continue; +/// } +/// let mut content = String::new(); +/// entry.read_to_string(&mut content).unwrap(); +/// match proc_macro2::TokenStream::from_str(&content) { +/// Ok(tokens) => {/* ... */}, +/// Err(_) => continue, +/// } +/// } +/// }); +/// } +/// }); +/// # +/// # fn every_version_of_every_crate() -> Vec { +/// # Vec::new() +/// # } +/// ``` +/// +/// # Panics +/// +/// This function is not applicable to and will panic if called from a +/// procedural macro. +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub fn invalidate_current_thread_spans() { + crate::imp::invalidate_current_thread_spans(); +} + +/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together +/// in a more compact representation than holding those 2 spans individually. +/// +/// [`Group`]: crate::Group +#[derive(Copy, Clone)] +pub struct DelimSpan { + inner: DelimSpanEnum, + _marker: ProcMacroAutoTraits, +} + +#[derive(Copy, Clone)] +enum DelimSpanEnum { + #[cfg(wrap_proc_macro)] + Compiler { + join: proc_macro::Span, + open: proc_macro::Span, + close: proc_macro::Span, + }, + Fallback(fallback::Span), +} + +impl DelimSpan { + pub(crate) fn new(group: &imp::Group) -> Self { + #[cfg(wrap_proc_macro)] + let inner = match group { + imp::Group::Compiler(group) => DelimSpanEnum::Compiler { + join: group.span(), + open: group.span_open(), + close: group.span_close(), + }, + imp::Group::Fallback(group) => DelimSpanEnum::Fallback(group.span()), + }; + + #[cfg(not(wrap_proc_macro))] + let inner = DelimSpanEnum::Fallback(group.span()); + + DelimSpan { + inner, + _marker: MARKER, + } + } + + /// Returns a span covering the entire delimited group. + pub fn join(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { join, .. } => Span::_new(imp::Span::Compiler(*join)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(*span), + } + } + + /// Returns a span for the opening punctuation of the group only. + pub fn open(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { open, .. } => Span::_new(imp::Span::Compiler(*open)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.first_byte()), + } + } + + /// Returns a span for the closing punctuation of the group only. + pub fn close(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { close, .. } => Span::_new(imp::Span::Compiler(*close)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.last_byte()), + } + } +} + +impl Debug for DelimSpan { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.join(), f) + } +} diff --git a/rust/proc-macro2/fallback.rs b/rust/proc-macro2/fallback.rs new file mode 100644 index 000000000..9b43c97df --- /dev/null +++ b/rust/proc-macro2/fallback.rs @@ -0,0 +1,1258 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(wrap_proc_macro)] +use crate::imp; +#[cfg(span_locations)] +use crate::location::LineColumn; +use crate::parse::{self, Cursor}; +use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; +use crate::{Delimiter, Spacing, TokenTree}; +#[cfg(all(span_locations, not(fuzzing)))] +use alloc::collections::BTreeMap; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cell::RefCell; +#[cfg(span_locations)] +use core::cmp; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display, Write}; +use core::mem::ManuallyDrop; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::ptr; +use core::str; +#[cfg(feature = "proc-macro")] +use core::str::FromStr; +use std::ffi::CStr; +#[cfg(wrap_proc_macro)] +use std::panic; +#[cfg(span_locations)] +use std::path::PathBuf; + +/// Force use of proc-macro2's fallback implementation of the API for now, even +/// if the compiler's implementation is available. +pub fn force() { + #[cfg(wrap_proc_macro)] + crate::detection::force_fallback(); +} + +/// Resume using the compiler's implementation of the proc macro API if it is +/// available. +pub fn unforce() { + #[cfg(wrap_proc_macro)] + crate::detection::unforce_fallback(); +} + +#[derive(Clone)] +pub(crate) struct TokenStream { + inner: RcVec, +} + +#[derive(Debug)] +pub(crate) struct LexError { + pub(crate) span: Span, +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn call_site() -> Self { + LexError { + span: Span::call_site(), + } + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + TokenStream { + inner: RcVecBuilder::new().build(), + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + // Create a dummy file & add it to the source map + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } + + parse::token_stream(cursor) + } + + #[cfg(feature = "proc-macro")] + pub(crate) fn from_str_unchecked(src: &str) -> Self { + Self::from_str_checked(src).unwrap() + } + + pub(crate) fn is_empty(&self) -> bool { + self.inner.len() == 0 + } + + fn take_inner(self) -> RcVecBuilder { + let nodrop = ManuallyDrop::new(self); + unsafe { ptr::read(&nodrop.inner) }.make_owned() + } +} + +fn push_token_from_proc_macro(mut vec: RcVecMut, token: TokenTree) { + // https://github.com/dtolnay/proc-macro2/issues/235 + match token { + TokenTree::Literal(crate::Literal { + #[cfg(wrap_proc_macro)] + inner: crate::imp::Literal::Fallback(literal), + #[cfg(not(wrap_proc_macro))] + inner: literal, + .. + }) if literal.repr.starts_with('-') => { + push_negative_literal(vec, literal); + } + _ => vec.push(token), + } + + #[cold] + fn push_negative_literal(mut vec: RcVecMut, mut literal: Literal) { + literal.repr.remove(0); + let mut punct = crate::Punct::new('-', Spacing::Alone); + punct.set_span(crate::Span::_new_fallback(literal.span)); + vec.push(TokenTree::Punct(punct)); + vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); + } +} + +// Nonrecursive to prevent stack overflow. +impl Drop for TokenStream { + fn drop(&mut self) { + let mut stack = Vec::new(); + let mut current = match self.inner.get_mut() { + Some(inner) => inner.take().into_iter(), + None => return, + }; + loop { + while let Some(token) = current.next() { + let group = match token { + TokenTree::Group(group) => group.inner, + _ => continue, + }; + #[cfg(wrap_proc_macro)] + let group = match group { + crate::imp::Group::Fallback(group) => group, + crate::imp::Group::Compiler(_) => continue, + }; + let mut group = group; + if let Some(inner) = group.stream.inner.get_mut() { + stack.push(current); + current = inner.take().into_iter(); + } + } + match stack.pop() { + Some(next) => current = next, + None => return, + } + } + } +} + +pub(crate) struct TokenStreamBuilder { + inner: RcVecBuilder, +} + +impl TokenStreamBuilder { + pub(crate) fn new() -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::new(), + } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::with_capacity(cap), + } + } + + pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) { + self.inner.push(tt); + } + + pub(crate) fn build(self) -> TokenStream { + TokenStream { + inner: self.inner.build(), + } + } +} + +#[cfg(span_locations)] +fn get_cursor(src: &str) -> Cursor { + #[cfg(fuzzing)] + return Cursor { rest: src, off: 1 }; + + // Create a dummy file & add it to the source map + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let mut sm = sm.borrow_mut(); + let span = sm.add_file(src); + Cursor { + rest: src, + off: span.lo, + } + }) +} + +#[cfg(not(span_locations))] +fn get_cursor(src: &str) -> Cursor { + Cursor { rest: src } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("cannot parse string into token stream") + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut joint = false; + for (i, tt) in self.inner.iter().enumerate() { + if i != 0 && !joint { + write!(f, " ")?; + } + joint = false; + match tt { + TokenTree::Group(tt) => Display::fmt(tt, f), + TokenTree::Ident(tt) => Display::fmt(tt, f), + TokenTree::Punct(tt) => { + joint = tt.spacing() == Spacing::Joint; + Display::fmt(tt, f) + } + TokenTree::Literal(tt) => Display::fmt(tt, f), + }?; + } + + Ok(()) + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(feature = "proc-macro")] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +#[cfg(feature = "proc-macro")] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +impl From for TokenStream { + fn from(tree: TokenTree) -> Self { + let mut stream = RcVecBuilder::new(); + push_token_from_proc_macro(stream.as_mut(), tree); + TokenStream { + inner: stream.build(), + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(tokens: I) -> Self { + let mut stream = TokenStream::new(); + stream.extend(tokens); + stream + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut v = RcVecBuilder::new(); + + for stream in streams { + v.extend(stream.take_inner()); + } + + TokenStream { inner: v.build() } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, tokens: I) { + let mut vec = self.inner.make_mut(); + tokens + .into_iter() + .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token)); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.make_mut().extend(streams.into_iter().flatten()); + } +} + +pub(crate) type TokenTreeIter = RcVecIntoIter; + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + self.take_inner().into_iter() + } +} + +#[cfg(all(span_locations, not(fuzzing)))] +thread_local! { + static SOURCE_MAP: RefCell = RefCell::new(SourceMap { + // Start with a single dummy file which all call_site() and def_site() + // spans reference. + files: vec![FileInfo { + source_text: String::new(), + span: Span { lo: 0, hi: 0 }, + lines: vec![0], + char_index_to_byte_offset: BTreeMap::new(), + }], + }); +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1)); +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct FileInfo { + source_text: String, + span: Span, + lines: Vec, + char_index_to_byte_offset: BTreeMap, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl FileInfo { + fn offset_line_column(&self, offset: usize) -> LineColumn { + assert!(self.span_within(Span { + lo: offset as u32, + hi: offset as u32, + })); + let offset = offset - self.span.lo as usize; + match self.lines.binary_search(&offset) { + Ok(found) => LineColumn { + line: found + 1, + column: 0, + }, + Err(idx) => LineColumn { + line: idx, + column: offset - self.lines[idx - 1], + }, + } + } + + fn span_within(&self, span: Span) -> bool { + span.lo >= self.span.lo && span.hi <= self.span.hi + } + + fn byte_range(&mut self, span: Span) -> Range { + let lo_char = (span.lo - self.span.lo) as usize; + + // Look up offset of the largest already-computed char index that is + // less than or equal to the current requested one. We resume counting + // chars from that point. + let (&last_char_index, &last_byte_offset) = self + .char_index_to_byte_offset + .range(..=lo_char) + .next_back() + .unwrap_or((&0, &0)); + + let lo_byte = if last_char_index == lo_char { + last_byte_offset + } else { + let total_byte_offset = match self.source_text[last_byte_offset..] + .char_indices() + .nth(lo_char - last_char_index) + { + Some((additional_offset, _ch)) => last_byte_offset + additional_offset, + None => self.source_text.len(), + }; + self.char_index_to_byte_offset + .insert(lo_char, total_byte_offset); + total_byte_offset + }; + + let trunc_lo = &self.source_text[lo_byte..]; + let char_len = (span.hi - span.lo) as usize; + lo_byte..match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => lo_byte + offset, + None => self.source_text.len(), + } + } + + fn source_text(&mut self, span: Span) -> String { + let byte_range = self.byte_range(span); + self.source_text[byte_range].to_owned() + } +} + +/// Computes the offsets of each line in the given source string +/// and the total number of characters +#[cfg(all(span_locations, not(fuzzing)))] +fn lines_offsets(s: &str) -> (usize, Vec) { + let mut lines = vec![0]; + let mut total = 0; + + for ch in s.chars() { + total += 1; + if ch == '\n' { + lines.push(total); + } + } + + (total, lines) +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct SourceMap { + files: Vec, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl SourceMap { + fn next_start_pos(&self) -> u32 { + // Add 1 so there's always space between files. + // + // We'll always have at least 1 file, as we initialize our files list + // with a dummy file. + self.files.last().unwrap().span.hi + 1 + } + + fn add_file(&mut self, src: &str) -> Span { + let (len, lines) = lines_offsets(src); + let lo = self.next_start_pos(); + let span = Span { + lo, + hi: lo + (len as u32), + }; + + self.files.push(FileInfo { + source_text: src.to_owned(), + span, + lines, + // Populated lazily by source_text(). + char_index_to_byte_offset: BTreeMap::new(), + }); + + span + } + + fn find(&self, span: Span) -> usize { + match self.files.binary_search_by(|file| { + if file.span.hi < span.lo { + Ordering::Less + } else if file.span.lo > span.hi { + Ordering::Greater + } else { + assert!(file.span_within(span)); + Ordering::Equal + } + }) { + Ok(i) => i, + Err(_) => unreachable!("Invalid span with no related FileInfo!"), + } + } + + fn filepath(&self, span: Span) -> String { + let i = self.find(span); + if i == 0 { + "".to_owned() + } else { + format!("", i) + } + } + + fn fileinfo(&self, span: Span) -> &FileInfo { + let i = self.find(span); + &self.files[i] + } + + fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo { + let i = self.find(span); + &mut self.files[i] + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Span { + #[cfg(span_locations)] + pub(crate) lo: u32, + #[cfg(span_locations)] + pub(crate) hi: u32, +} + +impl Span { + #[cfg(not(span_locations))] + pub(crate) fn call_site() -> Self { + Span {} + } + + #[cfg(span_locations)] + pub(crate) fn call_site() -> Self { + Span { lo: 0, hi: 0 } + } + + pub(crate) fn mixed_site() -> Self { + Span::call_site() + } + + #[cfg(procmacro2_semver_exempt)] + pub(crate) fn def_site() -> Self { + Span::call_site() + } + + pub(crate) fn resolved_at(&self, _other: Span) -> Span { + // Stable spans consist only of line/column information, so + // `resolved_at` and `located_at` only select which span the + // caller wants line/column information from. + *self + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + other + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + #[cfg(fuzzing)] + return 0..0; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + 0..0 + } else { + SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self)) + } + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.lo as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.hi as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + #[cfg(fuzzing)] + return "".to_owned(); + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + sm.filepath(*self) + }) + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + None + } + + #[cfg(not(span_locations))] + pub(crate) fn join(&self, _other: Span) -> Option { + Some(Span {}) + } + + #[cfg(span_locations)] + pub(crate) fn join(&self, other: Span) -> Option { + #[cfg(fuzzing)] + return { + let _ = other; + None + }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + // If `other` is not within the same FileInfo as us, return None. + if !sm.fileinfo(*self).span_within(other) { + return None; + } + Some(Span { + lo: cmp::min(self.lo, other.lo), + hi: cmp::max(self.hi, other.hi), + }) + }) + } + + #[cfg(not(span_locations))] + pub(crate) fn source_text(&self) -> Option { + None + } + + #[cfg(span_locations)] + pub(crate) fn source_text(&self) -> Option { + #[cfg(fuzzing)] + return None; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + None + } else { + Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + } + } + } + + #[cfg(not(span_locations))] + pub(crate) fn first_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn first_byte(self) -> Self { + Span { + lo: self.lo, + hi: cmp::min(self.lo.saturating_add(1), self.hi), + } + } + + #[cfg(not(span_locations))] + pub(crate) fn last_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn last_byte(self) -> Self { + Span { + lo: cmp::max(self.hi.saturating_sub(1), self.lo), + hi: self.hi, + } + } + + #[cfg(span_locations)] + fn is_call_site(&self) -> bool { + self.lo == 0 && self.hi == 0 + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + #[cfg(span_locations)] + return write!(f, "bytes({}..{})", self.lo, self.hi); + + #[cfg(not(span_locations))] + write!(f, "Span") + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + #[cfg(span_locations)] + { + if span.is_call_site() { + return; + } + } + + if cfg!(span_locations) { + debug.field("span", &span); + } +} + +#[derive(Clone)] +pub(crate) struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + delimiter, + stream, + span: Span::call_site(), + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + self.delimiter + } + + pub(crate) fn stream(&self) -> TokenStream { + self.stream.clone() + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn span_open(&self) -> Span { + self.span.first_byte() + } + + pub(crate) fn span_close(&self) -> Span { + self.span.last_byte() + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl Display for Group { + // We attempt to match libproc_macro's formatting. + // Empty parens: () + // Nonempty parens: (...) + // Empty brackets: [] + // Nonempty brackets: [...] + // Empty braces: { } + // Nonempty braces: { ... } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (open, close) = match self.delimiter { + Delimiter::Parenthesis => ("(", ")"), + Delimiter::Brace => ("{ ", "}"), + Delimiter::Bracket => ("[", "]"), + Delimiter::None => ("", ""), + }; + + f.write_str(open)?; + Display::fmt(&self.stream, f)?; + if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() { + f.write_str(" ")?; + } + f.write_str(close)?; + + Ok(()) + } +} + +impl Debug for Group { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Group"); + debug.field("delimiter", &self.delimiter); + debug.field("stream", &self.stream); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Ident { + sym: Box, + span: Span, + raw: bool, +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + validate_ident(string); + Ident::new_unchecked(string, span) + } + + pub(crate) fn new_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: false, + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + validate_ident_raw(string); + Ident::new_raw_unchecked(string, span) + } + + pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: true, + } + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +pub(crate) fn is_ident_start(c: char) -> bool { + c == '_' || c.is_ascii_alphabetic() +} + +pub(crate) fn is_ident_continue(c: char) -> bool { + c == '_' || c.is_ascii_alphanumeric() +} + +#[track_caller] +fn validate_ident(string: &str) { + if string.is_empty() { + panic!("Ident is not allowed to be empty; use Option"); + } + + if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') { + panic!("Ident cannot be a number; use Literal instead"); + } + + fn ident_ok(string: &str) -> bool { + let mut chars = string.chars(); + let first = chars.next().unwrap(); + if !is_ident_start(first) { + return false; + } + for ch in chars { + if !is_ident_continue(ch) { + return false; + } + } + true + } + + if !ident_ok(string) { + panic!("{:?} is not a valid Ident", string); + } +} + +#[track_caller] +fn validate_ident_raw(string: &str) { + validate_ident(string); + + match string { + "_" | "super" | "self" | "Self" | "crate" => { + panic!("`r#{}` cannot be a raw identifier", string); + } + _ => {} + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.sym == other.sym && self.raw == other.raw + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + if self.raw { + other.starts_with("r#") && *self.sym == other[2..] + } else { + *self.sym == *other + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.raw { + f.write_str("r#")?; + } + Display::fmt(&self.sym, f) + } +} + +#[allow(clippy::missing_fields_in_debug)] +impl Debug for Ident { + // Ident(proc_macro), Ident(r#union) + #[cfg(not(span_locations))] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_tuple("Ident"); + debug.field(&format_args!("{}", self)); + debug.finish() + } + + // Ident { + // sym: proc_macro, + // span: bytes(128..138) + // } + #[cfg(span_locations)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", self)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Literal { + pub(crate) repr: String, + span: Span, +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(format!(concat!("{}", stringify!($kind)), n)) + } + )*) +} + +macro_rules! unsuffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(n.to_string()) + } + )*) +} + +impl Literal { + pub(crate) fn _new(repr: String) -> Self { + Literal { + repr, + span: Span::call_site(), + } + } + + pub(crate) fn from_str_checked(repr: &str) -> Result { + let mut cursor = get_cursor(repr); + #[cfg(span_locations)] + let lo = cursor.off; + + let negative = cursor.starts_with_char('-'); + if negative { + cursor = cursor.advance(1); + if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { + return Err(LexError::call_site()); + } + } + + if let Ok((rest, mut literal)) = parse::literal(cursor) { + if rest.is_empty() { + if negative { + literal.repr.insert(0, '-'); + } + literal.span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + return Ok(literal); + } + } + Err(LexError::call_site()) + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(repr.to_owned()) + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_numbers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn string(string: &str) -> Literal { + let mut repr = String::with_capacity(string.len() + 2); + repr.push('"'); + escape_utf8(string, &mut repr); + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn character(ch: char) -> Literal { + let mut repr = String::new(); + repr.push('\''); + if ch == '"' { + // escape_debug turns this into '\"' which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + let mut repr = "b'".to_string(); + #[allow(clippy::match_overlapping_arm)] + match byte { + b'\0' => repr.push_str(r"\0"), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'\'' => repr.push_str(r"\'"), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(byte as char), + _ => { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + let mut repr = "b\"".to_string(); + let mut bytes = bytes.iter(); + while let Some(&b) = bytes.next() { + #[allow(clippy::match_overlapping_arm)] + match b { + b'\0' => repr.push_str(match bytes.as_slice().first() { + // circumvent clippy::octal_escapes lint + Some(b'0'..=b'7') => r"\x00", + _ => r"\0", + }), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'"' => repr.push_str("\\\""), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(b as char), + _ => { + let _ = write!(repr, r"\x{:02X}", b); + } + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + let mut repr = "c\"".to_string(); + let mut bytes = string.to_bytes(); + while !bytes.is_empty() { + let (valid, invalid) = match str::from_utf8(bytes) { + Ok(all_valid) => { + bytes = b""; + (all_valid, bytes) + } + Err(utf8_error) => { + let (valid, rest) = bytes.split_at(utf8_error.valid_up_to()); + let valid = str::from_utf8(valid).unwrap(); + let invalid = utf8_error + .error_len() + .map_or(rest, |error_len| &rest[..error_len]); + bytes = &bytes[valid.len() + invalid.len()..]; + (valid, invalid) + } + }; + escape_utf8(valid, &mut repr); + for &byte in invalid { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + #[cfg(not(span_locations))] + { + let _ = range; + None + } + + #[cfg(span_locations)] + { + use core::ops::Bound; + + let lo = match range.start_bound() { + Bound::Included(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)? + } + Bound::Excluded(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)?.checked_add(1)? + } + Bound::Unbounded => self.span.lo, + }; + let hi = match range.end_bound() { + Bound::Included(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)?.checked_add(1)? + } + Bound::Excluded(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)? + } + Bound::Unbounded => self.span.hi, + }; + if lo <= hi && hi <= self.span.hi { + Some(Span { lo, hi }) + } else { + None + } + } + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.repr, f) + } +} + +impl Debug for Literal { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Literal"); + debug.field("lit", &format_args!("{}", self.repr)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +fn escape_utf8(string: &str, repr: &mut String) { + let mut chars = string.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + r"\x00" + } else { + r"\0" + }, + ); + } else if ch == '\'' { + // escape_debug turns this into "\'" which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + } +} + +#[cfg(feature = "proc-macro")] +pub(crate) trait FromStr2: FromStr { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool; + + #[cfg(wrap_proc_macro)] + fn from_str_checked(src: &str) -> Result { + // Validate using fallback parser, because rustc is incapable of + // returning a recoverable Err for certain invalid token streams, and + // will instead permanently poison the compilation. + if !Self::valid(src) { + return Err(imp::LexError::CompilerPanic); + } + + // Catch panic to work around https://github.com/rust-lang/rust/issues/58736. + match panic::catch_unwind(|| Self::from_str(src)) { + Ok(Ok(ok)) => Ok(ok), + Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)), + Err(_panic) => Err(imp::LexError::CompilerPanic), + } + } + + fn from_str_unchecked(src: &str) -> Self { + Self::from_str(src).unwrap() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::TokenStream { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + TokenStream::from_str_checked(src).is_ok() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::Literal { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + Literal::from_str_checked(src).is_ok() + } +} diff --git a/rust/proc-macro2/lib.rs b/rust/proc-macro2/lib.rs new file mode 100644 index 000000000..5d408943f --- /dev/null +++ b/rust/proc-macro2/lib.rs @@ -0,0 +1,1355 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! [![github]](https://github.com/dtolnay/proc-macro2) [![crates-io]](https://crates.io/crates/proc-macro2) [![docs-rs]](crate) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! A wrapper around the procedural macro API of the compiler's [`proc_macro`] +//! crate. This library serves two purposes: +//! +//! - **Bring proc-macro-like functionality to other contexts like build.rs and +//! main.rs.** Types from `proc_macro` are entirely specific to procedural +//! macros and cannot ever exist in code outside of a procedural macro. +//! Meanwhile `proc_macro2` types may exist anywhere including non-macro code. +//! By developing foundational libraries like [syn] and [quote] against +//! `proc_macro2` rather than `proc_macro`, the procedural macro ecosystem +//! becomes easily applicable to many other use cases and we avoid +//! reimplementing non-macro equivalents of those libraries. +//! +//! - **Make procedural macros unit testable.** As a consequence of being +//! specific to procedural macros, nothing that uses `proc_macro` can be +//! executed from a unit test. In order for helper libraries or components of +//! a macro to be testable in isolation, they must be implemented using +//! `proc_macro2`. +//! +//! [syn]: https://github.com/dtolnay/syn +//! [quote]: https://github.com/dtolnay/quote +//! +//! # Usage +//! +//! The skeleton of a typical procedural macro typically looks like this: +//! +//! ``` +//! extern crate proc_macro; +//! +//! # const IGNORE: &str = stringify! { +//! #[proc_macro_derive(MyDerive)] +//! # }; +//! # #[cfg(wrap_proc_macro)] +//! pub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +//! let input = proc_macro2::TokenStream::from(input); +//! +//! let output: proc_macro2::TokenStream = { +//! /* transform input */ +//! # input +//! }; +//! +//! proc_macro::TokenStream::from(output) +//! } +//! ``` +//! +//! If parsing with [Syn], you'll use [`parse_macro_input!`] instead to +//! propagate parse errors correctly back to the compiler when parsing fails. +//! +//! [`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html +//! +//! # Unstable features +//! +//! The default feature set of proc-macro2 tracks the most recent stable +//! compiler API. Functionality in `proc_macro` that is not yet stable is not +//! exposed by proc-macro2 by default. +//! +//! To opt into the additional APIs available in the most recent nightly +//! compiler, the `procmacro2_semver_exempt` config flag must be passed to +//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As +//! these are unstable APIs that track the nightly compiler, minor versions of +//! proc-macro2 may make breaking changes to them at any time. +//! +//! ```sh +//! RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build +//! ``` +//! +//! Note that this must not only be done for your crate, but for any crate that +//! depends on your crate. This infectious nature is intentional, as it serves +//! as a reminder that you are outside of the normal semver guarantees. +//! +//! Semver exempt methods are marked as such in the proc-macro2 documentation. +//! +//! # Thread-Safety +//! +//! Most types in this crate are `!Sync` because the underlying compiler +//! types make use of thread-local memory, meaning they cannot be accessed from +//! a different thread. + +// Proc-macro2 types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.101")] +#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] +#![cfg_attr(super_unstable, feature(proc_macro_def_site))] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(unsafe_op_in_unsafe_fn)] +#![allow( + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::checked_conversions, + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::incompatible_msrv, + clippy::items_after_statements, + clippy::iter_without_into_iter, + clippy::let_underscore_untyped, + clippy::manual_assert, + clippy::manual_range_contains, + clippy::missing_panics_doc, + clippy::missing_safety_doc, + clippy::must_use_candidate, + clippy::needless_doctest_main, + clippy::needless_lifetimes, + clippy::new_without_default, + clippy::return_self_not_must_use, + clippy::shadow_unrelated, + clippy::trivially_copy_pass_by_ref, + clippy::unnecessary_wraps, + clippy::unused_self, + clippy::used_underscore_binding, + clippy::vec_init_then_push +)] +#![allow(unknown_lints, mismatched_lifetime_syntaxes)] + +#[cfg(all(procmacro2_semver_exempt, wrap_proc_macro, not(super_unstable)))] +compile_error! {"\ + Something is not right. If you've tried to turn on \ + procmacro2_semver_exempt, you need to ensure that it \ + is turned on for the compilation of the proc-macro2 \ + build script as well. +"} + +#[cfg(all( + procmacro2_nightly_testing, + feature = "proc-macro", + not(proc_macro_span) +))] +compile_error! {"\ + Build script probe failed to compile. +"} + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod marker; +mod parse; +mod probe; +mod rcvec; + +#[cfg(wrap_proc_macro)] +mod detection; + +// Public for proc_macro2::fallback::force() and unforce(), but those are quite +// a niche use case so we omit it from rustdoc. +#[doc(hidden)] +pub mod fallback; + +pub mod extra; + +#[cfg(not(wrap_proc_macro))] +use crate::fallback as imp; +#[path = "wrapper.rs"] +#[cfg(wrap_proc_macro)] +mod imp; + +#[cfg(span_locations)] +mod location; + +use crate::extra::DelimSpan; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display}; +use core::hash::{Hash, Hasher}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::str::FromStr; +use std::error::Error; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub use crate::location::LineColumn; + +/// An abstract stream of tokens, or more concretely a sequence of token trees. +/// +/// This type provides interfaces for iterating over token trees and for +/// collecting token trees into one stream. +/// +/// Token stream is both the input and output of `#[proc_macro]`, +/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions. +#[derive(Clone)] +pub struct TokenStream { + inner: imp::TokenStream, + _marker: ProcMacroAutoTraits, +} + +/// Error returned from `TokenStream::from_str`. +pub struct LexError { + inner: imp::LexError, + _marker: ProcMacroAutoTraits, +} + +impl TokenStream { + fn _new(inner: imp::TokenStream) -> Self { + TokenStream { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::TokenStream) -> Self { + TokenStream { + inner: imp::TokenStream::from(inner), + _marker: MARKER, + } + } + + /// Returns an empty `TokenStream` containing no token trees. + pub fn new() -> Self { + TokenStream::_new(imp::TokenStream::new()) + } + + /// Checks if this `TokenStream` is empty. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + +/// `TokenStream::default()` returns an empty stream, +/// i.e. this is equivalent with `TokenStream::new()`. +impl Default for TokenStream { + fn default() -> Self { + TokenStream::new() + } +} + +/// Attempts to break the string into tokens and parse those tokens into a token +/// stream. +/// +/// May fail for a number of reasons, for example, if the string contains +/// unbalanced delimiters or characters not existing in the language. +/// +/// NOTE: Some errors may cause panics instead of returning `LexError`. We +/// reserve the right to change these errors into `LexError`s later. +impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + match imp::TokenStream::from_str_checked(src) { + Ok(tokens) => Ok(TokenStream::_new(tokens)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::_new(imp::TokenStream::from(inner)) + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from(inner.inner) + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + TokenStream::_new(imp::TokenStream::from(token)) + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.extend(streams); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner + .extend(streams.into_iter().map(|stream| stream.inner)); + } +} + +/// Collects a number of token trees into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().collect()) + } +} +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().map(|i| i.inner).collect()) + } +} + +/// Prints the token stream as a string that is supposed to be losslessly +/// convertible back into the same token stream (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Prints token in a form convenient for debugging. +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl LexError { + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Error for LexError {} + +/// A region of source code, along with macro expansion information. +#[derive(Copy, Clone)] +pub struct Span { + inner: imp::Span, + _marker: ProcMacroAutoTraits, +} + +impl Span { + fn _new(inner: imp::Span) -> Self { + Span { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Span) -> Self { + Span { + inner: imp::Span::from(inner), + _marker: MARKER, + } + } + + /// The span of the invocation of the current procedural macro. + /// + /// Identifiers created with this span will be resolved as if they were + /// written directly at the macro call location (call-site hygiene) and + /// other code at the macro call site will be able to refer to them as well. + pub fn call_site() -> Self { + Span::_new(imp::Span::call_site()) + } + + /// The span located at the invocation of the procedural macro, but with + /// local variables, labels, and `$crate` resolved at the definition site + /// of the macro. This is the same hygiene behavior as `macro_rules`. + pub fn mixed_site() -> Self { + Span::_new(imp::Span::mixed_site()) + } + + /// A span that resolves at the macro definition site. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn def_site() -> Self { + Span::_new(imp::Span::def_site()) + } + + /// Creates a new span with the same line/column information as `self` but + /// that resolves symbols as though it were at `other`. + pub fn resolved_at(&self, other: Span) -> Span { + Span::_new(self.inner.resolved_at(other.inner)) + } + + /// Creates a new span with the same name resolution behavior as `self` but + /// with the line/column information of `other`. + pub fn located_at(&self, other: Span) -> Span { + Span::_new(self.inner.located_at(other.inner)) + } + + /// Convert `proc_macro2::Span` to `proc_macro::Span`. + /// + /// This method is available when building with a nightly compiler, or when + /// building with rustc 1.29+ *without* semver exempt features. + /// + /// # Panics + /// + /// Panics if called from outside of a procedural macro. Unlike + /// `proc_macro2::Span`, the `proc_macro::Span` type can only exist within + /// the context of a procedural macro invocation. + #[cfg(wrap_proc_macro)] + pub fn unwrap(self) -> proc_macro::Span { + self.inner.unwrap() + } + + // Soft deprecated. Please use Span::unwrap. + #[cfg(wrap_proc_macro)] + #[doc(hidden)] + pub fn unstable(self) -> proc_macro::Span { + self.unwrap() + } + + /// Returns the span's byte position range in the source file. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned range is only + /// accurate if compiled with a nightly toolchain. The stable toolchain does + /// not have this information available. When executing outside of a + /// procedural macro, such as main.rs or build.rs, the byte range is always + /// accurate regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn byte_range(&self) -> Range { + self.inner.byte_range() + } + + /// Get the starting line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn start(&self) -> LineColumn { + self.inner.start() + } + + /// Get the ending line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn end(&self) -> LineColumn { + self.inner.end() + } + + /// The path to the source file in which this span occurs, for display + /// purposes. + /// + /// This might not correspond to a valid file system path. It might be + /// remapped, or might be an artificial path such as `""`. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn file(&self) -> String { + self.inner.file() + } + + /// The path to the source file in which this span occurs on disk. + /// + /// This is the actual path on disk. It is unaffected by path remapping. + /// + /// This path should not be embedded in the output of the macro; prefer + /// `file()` instead. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn local_file(&self) -> Option { + self.inner.local_file() + } + + /// Create a new span encompassing `self` and `other`. + /// + /// Returns `None` if `self` and `other` are from different files. + /// + /// Warning: the underlying [`proc_macro::Span::join`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn join(&self, other: Span) -> Option { + self.inner.join(other.inner).map(Span::_new) + } + + /// Compares two spans to see if they're equal. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn eq(&self, other: &Span) -> bool { + self.inner.eq(&other.inner) + } + + /// Returns the source text behind a span. This preserves the original + /// source code, including spaces and comments. It only returns a result if + /// the span corresponds to real source code. + /// + /// Note: The observable result of a macro should only rely on the tokens + /// and not on this source text. The result of this function is a best + /// effort to be used for diagnostics only. + pub fn source_text(&self) -> Option { + self.inner.source_text() + } +} + +/// Prints a span in a form convenient for debugging. +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`). +#[derive(Clone)] +pub enum TokenTree { + /// A token stream surrounded by bracket delimiters. + Group(Group), + /// An identifier. + Ident(Ident), + /// A single punctuation character (`+`, `,`, `$`, etc.). + Punct(Punct), + /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc. + Literal(Literal), +} + +impl TokenTree { + /// Returns the span of this tree, delegating to the `span` method of + /// the contained token or a delimited stream. + pub fn span(&self) -> Span { + match self { + TokenTree::Group(t) => t.span(), + TokenTree::Ident(t) => t.span(), + TokenTree::Punct(t) => t.span(), + TokenTree::Literal(t) => t.span(), + } + } + + /// Configures the span for *only this token*. + /// + /// Note that if this token is a `Group` then this method will not configure + /// the span of each of the internal tokens, this will simply delegate to + /// the `set_span` method of each variant. + pub fn set_span(&mut self, span: Span) { + match self { + TokenTree::Group(t) => t.set_span(span), + TokenTree::Ident(t) => t.set_span(span), + TokenTree::Punct(t) => t.set_span(span), + TokenTree::Literal(t) => t.set_span(span), + } + } +} + +impl From for TokenTree { + fn from(g: Group) -> Self { + TokenTree::Group(g) + } +} + +impl From for TokenTree { + fn from(g: Ident) -> Self { + TokenTree::Ident(g) + } +} + +impl From for TokenTree { + fn from(g: Punct) -> Self { + TokenTree::Punct(g) + } +} + +impl From for TokenTree { + fn from(g: Literal) -> Self { + TokenTree::Literal(g) + } +} + +/// Prints the token tree as a string that is supposed to be losslessly +/// convertible back into the same token tree (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenTree::Group(t) => Display::fmt(t, f), + TokenTree::Ident(t) => Display::fmt(t, f), + TokenTree::Punct(t) => Display::fmt(t, f), + TokenTree::Literal(t) => Display::fmt(t, f), + } + } +} + +/// Prints token tree in a form convenient for debugging. +impl Debug for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Each of these has the name in the struct type in the derived debug, + // so don't bother with an extra layer of indirection + match self { + TokenTree::Group(t) => Debug::fmt(t, f), + TokenTree::Ident(t) => { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", t)); + imp::debug_span_field_if_nontrivial(&mut debug, t.span().inner); + debug.finish() + } + TokenTree::Punct(t) => Debug::fmt(t, f), + TokenTree::Literal(t) => Debug::fmt(t, f), + } + } +} + +/// A delimited token stream. +/// +/// A `Group` internally contains a `TokenStream` which is surrounded by +/// `Delimiter`s. +#[derive(Clone)] +pub struct Group { + inner: imp::Group, +} + +/// Describes how a sequence of token trees is delimited. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Delimiter { + /// `( ... )` + Parenthesis, + /// `{ ... }` + Brace, + /// `[ ... ]` + Bracket, + /// `∅ ... ∅` + /// + /// An invisible delimiter, that may, for example, appear around tokens + /// coming from a "macro variable" `$var`. It is important to preserve + /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`. + /// Invisible delimiters may not survive roundtrip of a token stream through + /// a string. + /// + ///

+ /// + /// Note: rustc currently can ignore the grouping of tokens delimited by `None` in the output + /// of a proc_macro. Only `None`-delimited groups created by a macro_rules macro in the input + /// of a proc_macro macro are preserved, and only in very specific circumstances. + /// Any `None`-delimited groups (re)created by a proc_macro will therefore not preserve + /// operator priorities as indicated above. The other `Delimiter` variants should be used + /// instead in this context. This is a rustc bug. For details, see + /// [rust-lang/rust#67062](https://github.com/rust-lang/rust/issues/67062). + /// + ///
+ None, +} + +impl Group { + fn _new(inner: imp::Group) -> Self { + Group { inner } + } + + fn _new_fallback(inner: fallback::Group) -> Self { + Group { + inner: imp::Group::from(inner), + } + } + + /// Creates a new `Group` with the given delimiter and token stream. + /// + /// This constructor will set the span for this group to + /// `Span::call_site()`. To change the span you can use the `set_span` + /// method below. + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + inner: imp::Group::new(delimiter, stream.inner), + } + } + + /// Returns the punctuation used as the delimiter for this group: a set of + /// parentheses, square brackets, or curly braces. + pub fn delimiter(&self) -> Delimiter { + self.inner.delimiter() + } + + /// Returns the `TokenStream` of tokens that are delimited in this `Group`. + /// + /// Note that the returned token stream does not include the delimiter + /// returned above. + pub fn stream(&self) -> TokenStream { + TokenStream::_new(self.inner.stream()) + } + + /// Returns the span for the delimiters of this token stream, spanning the + /// entire `Group`. + /// + /// ```text + /// pub fn span(&self) -> Span { + /// ^^^^^^^ + /// ``` + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Returns the span pointing to the opening delimiter of this group. + /// + /// ```text + /// pub fn span_open(&self) -> Span { + /// ^ + /// ``` + pub fn span_open(&self) -> Span { + Span::_new(self.inner.span_open()) + } + + /// Returns the span pointing to the closing delimiter of this group. + /// + /// ```text + /// pub fn span_close(&self) -> Span { + /// ^ + /// ``` + pub fn span_close(&self) -> Span { + Span::_new(self.inner.span_close()) + } + + /// Returns an object that holds this group's `span_open()` and + /// `span_close()` together (in a more compact representation than holding + /// those 2 spans individually). + pub fn delim_span(&self) -> DelimSpan { + DelimSpan::new(&self.inner) + } + + /// Configures the span for this `Group`'s delimiters, but not its internal + /// tokens. + /// + /// This method will **not** set the span of all the internal tokens spanned + /// by this group, but rather it will only set the span of the delimiter + /// tokens at the level of the `Group`. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +/// Prints the group as a string that should be losslessly convertible back +/// into the same group (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters. +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, formatter) + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, formatter) + } +} + +/// A `Punct` is a single punctuation character like `+`, `-` or `#`. +/// +/// Multicharacter operators like `+=` are represented as two instances of +/// `Punct` with different forms of `Spacing` returned. +#[derive(Clone)] +pub struct Punct { + ch: char, + spacing: Spacing, + span: Span, +} + +/// Whether a `Punct` is followed immediately by another `Punct` or followed by +/// another token or whitespace. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Spacing { + /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`. + Alone, + /// E.g. `+` is `Joint` in `+=` or `'` is `Joint` in `'#`. + /// + /// Additionally, single quote `'` can join with identifiers to form + /// lifetimes `'ident`. + Joint, +} + +impl Punct { + /// Creates a new `Punct` from the given character and spacing. + /// + /// The `ch` argument must be a valid punctuation character permitted by the + /// language, otherwise the function will panic. + /// + /// The returned `Punct` will have the default span of `Span::call_site()` + /// which can be further configured with the `set_span` method below. + pub fn new(ch: char, spacing: Spacing) -> Self { + if let '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' + | '<' | '=' | '>' | '?' | '@' | '^' | '|' | '~' = ch + { + Punct { + ch, + spacing, + span: Span::call_site(), + } + } else { + panic!("unsupported proc macro punctuation character {:?}", ch); + } + } + + /// Returns the value of this punctuation character as `char`. + pub fn as_char(&self) -> char { + self.ch + } + + /// Returns the spacing of this punctuation character, indicating whether + /// it's immediately followed by another `Punct` in the token stream, so + /// they can potentially be combined into a multicharacter operator + /// (`Joint`), or it's followed by some other token or whitespace (`Alone`) + /// so the operator has certainly ended. + pub fn spacing(&self) -> Spacing { + self.spacing + } + + /// Returns the span for this punctuation character. + pub fn span(&self) -> Span { + self.span + } + + /// Configure the span for this punctuation character. + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +/// Prints the punctuation character as a string that should be losslessly +/// convertible back into the same character. +impl Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.ch, f) + } +} + +impl Debug for Punct { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Punct"); + debug.field("char", &self.ch); + debug.field("spacing", &self.spacing); + imp::debug_span_field_if_nontrivial(&mut debug, self.span.inner); + debug.finish() + } +} + +/// A word of Rust code, which may be a keyword or legal variable name. +/// +/// An identifier consists of at least one Unicode code point, the first of +/// which has the XID_Start property and the rest of which have the XID_Continue +/// property. +/// +/// - The empty string is not an identifier. Use `Option`. +/// - A lifetime is not an identifier. Use `syn::Lifetime` instead. +/// +/// An identifier constructed with `Ident::new` is permitted to be a Rust +/// keyword, though parsing one through its [`Parse`] implementation rejects +/// Rust keywords. Use `input.call(Ident::parse_any)` when parsing to match the +/// behaviour of `Ident::new`. +/// +/// [`Parse`]: https://docs.rs/syn/2.0/syn/parse/trait.Parse.html +/// +/// # Examples +/// +/// A new ident can be created from a string using the `Ident::new` function. +/// A span must be provided explicitly which governs the name resolution +/// behavior of the resulting identifier. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// +/// fn main() { +/// let call_ident = Ident::new("calligraphy", Span::call_site()); +/// +/// println!("{}", call_ident); +/// } +/// ``` +/// +/// An ident can be interpolated into a token stream using the `quote!` macro. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// use quote::quote; +/// +/// fn main() { +/// let ident = Ident::new("demo", Span::call_site()); +/// +/// // Create a variable binding whose name is this ident. +/// let expanded = quote! { let #ident = 10; }; +/// +/// // Create a variable binding with a slightly different name. +/// let temp_ident = Ident::new(&format!("new_{}", ident), Span::call_site()); +/// let expanded = quote! { let #temp_ident = 10; }; +/// } +/// ``` +/// +/// A string representation of the ident is available through the `to_string()` +/// method. +/// +/// ``` +/// # use proc_macro2::{Ident, Span}; +/// # +/// # let ident = Ident::new("another_identifier", Span::call_site()); +/// # +/// // Examine the ident as a string. +/// let ident_string = ident.to_string(); +/// if ident_string.len() > 60 { +/// println!("Very long identifier: {}", ident_string) +/// } +/// ``` +#[derive(Clone)] +pub struct Ident { + inner: imp::Ident, + _marker: ProcMacroAutoTraits, +} + +impl Ident { + fn _new(inner: imp::Ident) -> Self { + Ident { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Ident) -> Self { + Ident { + inner: imp::Ident::from(inner), + _marker: MARKER, + } + } + + /// Creates a new `Ident` with the given `string` as well as the specified + /// `span`. + /// + /// The `string` argument must be a valid identifier permitted by the + /// language, otherwise the function will panic. + /// + /// Note that `span`, currently in rustc, configures the hygiene information + /// for this identifier. + /// + /// As of this time `Span::call_site()` explicitly opts-in to "call-site" + /// hygiene meaning that identifiers created with this span will be resolved + /// as if they were written directly at the location of the macro call, and + /// other code at the macro call site will be able to refer to them as well. + /// + /// Later spans like `Span::def_site()` will allow to opt-in to + /// "definition-site" hygiene meaning that identifiers created with this + /// span will be resolved at the location of the macro definition and other + /// code at the macro call site will not be able to refer to them. + /// + /// Due to the current importance of hygiene this constructor, unlike other + /// tokens, requires a `Span` to be specified at construction. + /// + /// # Panics + /// + /// Panics if the input string is neither a keyword nor a legal variable + /// name. If you are not sure whether the string contains an identifier and + /// need to handle an error case, use + /// syn::parse_str::<Ident> + /// rather than `Ident::new`. + #[track_caller] + pub fn new(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_checked(string, span.inner)) + } + + /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). The + /// `string` argument must be a valid identifier permitted by the language + /// (including keywords, e.g. `fn`). Keywords which are usable in path + /// segments (e.g. `self`, `super`) are not supported, and will cause a + /// panic. + #[track_caller] + pub fn new_raw(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_raw_checked(string, span.inner)) + } + + /// Returns the span of this `Ident`. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span of this `Ident`, possibly changing its hygiene + /// context. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.inner == other.inner + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + self.inner == other + } +} + +impl Eq for Ident {} + +impl PartialOrd for Ident { + fn partial_cmp(&self, other: &Ident) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Ident { + fn cmp(&self, other: &Ident) -> Ordering { + self.to_string().cmp(&other.to_string()) + } +} + +impl Hash for Ident { + fn hash(&self, hasher: &mut H) { + self.to_string().hash(hasher); + } +} + +/// Prints the identifier as a string that should be losslessly convertible back +/// into the same identifier. +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A literal string (`"hello"`), byte string (`b"hello"`), character (`'a'`), +/// byte character (`b'a'`), an integer or floating point number with or without +/// a suffix (`1`, `1u8`, `2.3`, `2.3f32`). +/// +/// Boolean literals like `true` and `false` do not belong here, they are +/// `Ident`s. +#[derive(Clone)] +pub struct Literal { + inner: imp::Literal, + _marker: ProcMacroAutoTraits, +} + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new suffixed integer literal with the specified value. + /// + /// This function will create an integer like `1u32` where the integer + /// value specified is the first part of the token and the integral is + /// also suffixed at the end. Literals created from negative numbers may + /// not survive roundtrips through `TokenStream` or strings and may be + /// broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new unsuffixed integer literal with the specified value. + /// + /// This function will create an integer like `1` where the integer + /// value specified is the first part of the token. No suffix is + /// specified on this token, meaning that invocations like + /// `Literal::i8_unsuffixed(1)` are equivalent to + /// `Literal::u32_unsuffixed(1)`. Literals created from negative numbers + /// may not survive roundtrips through `TokenStream` or strings and may + /// be broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +impl Literal { + fn _new(inner: imp::Literal) -> Self { + Literal { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Literal) -> Self { + Literal { + inner: imp::Literal::from(inner), + _marker: MARKER, + } + } + + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + } + + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_unsuffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f64` where the value + /// specified is the preceding part of the token and `f64` is the suffix of + /// the token. This token will always be inferred to be an `f64` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_suffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_suffixed(f)) + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_unsuffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f32` where the value + /// specified is the preceding part of the token and `f32` is the suffix of + /// the token. This token will always be inferred to be an `f32` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_suffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_suffixed(f)) + } + + /// String literal. + pub fn string(string: &str) -> Literal { + Literal::_new(imp::Literal::string(string)) + } + + /// Character literal. + pub fn character(ch: char) -> Literal { + Literal::_new(imp::Literal::character(ch)) + } + + /// Byte character literal. + pub fn byte_character(byte: u8) -> Literal { + Literal::_new(imp::Literal::byte_character(byte)) + } + + /// Byte string literal. + pub fn byte_string(bytes: &[u8]) -> Literal { + Literal::_new(imp::Literal::byte_string(bytes)) + } + + /// C string literal. + pub fn c_string(string: &CStr) -> Literal { + Literal::_new(imp::Literal::c_string(string)) + } + + /// Returns the span encompassing this literal. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span associated for this literal. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } + + /// Returns a `Span` that is a subset of `self.span()` containing only + /// the source bytes in range `range`. Returns `None` if the would-be + /// trimmed span is outside the bounds of `self`. + /// + /// Warning: the underlying [`proc_macro::Literal::subspan`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn subspan>(&self, range: R) -> Option { + self.inner.subspan(range).map(Span::_new) + } + + // Intended for the `quote!` macro to use when constructing a proc-macro2 + // token out of a macro_rules $:literal token, which is already known to be + // a valid literal. This avoids reparsing/validating the literal's string + // representation. This is not public API other than for quote. + #[doc(hidden)] + pub unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(unsafe { imp::Literal::from_str_unchecked(repr) }) + } +} + +impl FromStr for Literal { + type Err = LexError; + + fn from_str(repr: &str) -> Result { + match imp::Literal::from_str_checked(repr) { + Ok(lit) => Ok(Literal::_new(lit)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +pub mod token_stream { + use crate::marker::{ProcMacroAutoTraits, MARKER}; + use crate::{imp, TokenTree}; + use core::fmt::{self, Debug}; + + pub use crate::TokenStream; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// + /// The iteration is "shallow", e.g. the iterator doesn't recurse into + /// delimited groups, and returns whole groups as token trees. + #[derive(Clone)] + pub struct IntoIter { + inner: imp::TokenTreeIter, + _marker: ProcMacroAutoTraits, + } + + impl Iterator for IntoIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + } + + impl Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } + } + + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { + inner: self.inner.into_iter(), + _marker: MARKER, + } + } + } +} diff --git a/rust/proc-macro2/location.rs b/rust/proc-macro2/location.rs new file mode 100644 index 000000000..3a11871e2 --- /dev/null +++ b/rust/proc-macro2/location.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use core::cmp::Ordering; + +/// A line-column pair representing the start or end of a `Span`. +/// +/// This type is semver exempt and not exposed by default. +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct LineColumn { + /// The 1-indexed line in the source file on which the span starts or ends + /// (inclusive). + pub line: usize, + /// The 0-indexed column (in UTF-8 characters) in the source file on which + /// the span starts or ends (inclusive). + pub column: usize, +} + +impl Ord for LineColumn { + fn cmp(&self, other: &Self) -> Ordering { + self.line + .cmp(&other.line) + .then(self.column.cmp(&other.column)) + } +} + +impl PartialOrd for LineColumn { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/rust/proc-macro2/marker.rs b/rust/proc-macro2/marker.rs new file mode 100644 index 000000000..6f3224248 --- /dev/null +++ b/rust/proc-macro2/marker.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::rc::Rc; +use core::marker::PhantomData; +use core::panic::{RefUnwindSafe, UnwindSafe}; + +// Zero sized marker with the correct set of autotrait impls we want all proc +// macro types to have. +#[derive(Copy, Clone)] +#[cfg_attr( + all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)), + derive(PartialEq, Eq) +)] +pub(crate) struct ProcMacroAutoTraits(PhantomData>); + +pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData); + +impl UnwindSafe for ProcMacroAutoTraits {} +impl RefUnwindSafe for ProcMacroAutoTraits {} diff --git a/rust/proc-macro2/parse.rs b/rust/proc-macro2/parse.rs new file mode 100644 index 000000000..a005dea1f --- /dev/null +++ b/rust/proc-macro2/parse.rs @@ -0,0 +1,997 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::fallback::{ + self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream, + TokenStreamBuilder, +}; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::char; +use core::str::{Bytes, CharIndices, Chars}; + +#[derive(Copy, Clone, Eq, PartialEq)] +pub(crate) struct Cursor<'a> { + pub(crate) rest: &'a str, + #[cfg(span_locations)] + pub(crate) off: u32, +} + +impl<'a> Cursor<'a> { + pub(crate) fn advance(&self, bytes: usize) -> Cursor<'a> { + let (_front, rest) = self.rest.split_at(bytes); + Cursor { + rest, + #[cfg(span_locations)] + off: self.off + _front.chars().count() as u32, + } + } + + pub(crate) fn starts_with(&self, s: &str) -> bool { + self.rest.starts_with(s) + } + + pub(crate) fn starts_with_char(&self, ch: char) -> bool { + self.rest.starts_with(ch) + } + + pub(crate) fn starts_with_fn(&self, f: Pattern) -> bool + where + Pattern: FnMut(char) -> bool, + { + self.rest.starts_with(f) + } + + pub(crate) fn is_empty(&self) -> bool { + self.rest.is_empty() + } + + fn len(&self) -> usize { + self.rest.len() + } + + fn as_bytes(&self) -> &'a [u8] { + self.rest.as_bytes() + } + + fn bytes(&self) -> Bytes<'a> { + self.rest.bytes() + } + + fn chars(&self) -> Chars<'a> { + self.rest.chars() + } + + fn char_indices(&self) -> CharIndices<'a> { + self.rest.char_indices() + } + + fn parse(&self, tag: &str) -> Result, Reject> { + if self.starts_with(tag) { + Ok(self.advance(tag.len())) + } else { + Err(Reject) + } + } +} + +pub(crate) struct Reject; +type PResult<'a, O> = Result<(Cursor<'a>, O), Reject>; + +fn skip_whitespace(input: Cursor) -> Cursor { + let mut s = input; + + while !s.is_empty() { + let byte = s.as_bytes()[0]; + if byte == b'/' { + if s.starts_with("//") + && (!s.starts_with("///") || s.starts_with("////")) + && !s.starts_with("//!") + { + let (cursor, _) = take_until_newline_or_eof(s); + s = cursor; + continue; + } else if s.starts_with("/**/") { + s = s.advance(4); + continue; + } else if s.starts_with("/*") + && (!s.starts_with("/**") || s.starts_with("/***")) + && !s.starts_with("/*!") + { + match block_comment(s) { + Ok((rest, _)) => { + s = rest; + continue; + } + Err(Reject) => return s, + } + } + } + match byte { + b' ' | 0x09..=0x0d => { + s = s.advance(1); + continue; + } + b if b.is_ascii() => {} + _ => { + let ch = s.chars().next().unwrap(); + if is_whitespace(ch) { + s = s.advance(ch.len_utf8()); + continue; + } + } + } + return s; + } + s +} + +fn block_comment(input: Cursor) -> PResult<&str> { + if !input.starts_with("/*") { + return Err(Reject); + } + + let mut depth = 0usize; + let bytes = input.as_bytes(); + let mut i = 0usize; + let upper = bytes.len() - 1; + + while i < upper { + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + depth += 1; + i += 1; // eat '*' + } else if bytes[i] == b'*' && bytes[i + 1] == b'/' { + depth -= 1; + if depth == 0 { + return Ok((input.advance(i + 2), &input.rest[..i + 2])); + } + i += 1; // eat '/' + } + i += 1; + } + + Err(Reject) +} + +fn is_whitespace(ch: char) -> bool { + // Rust treats left-to-right mark and right-to-left mark as whitespace + ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' +} + +fn word_break(input: Cursor) -> Result { + match input.chars().next() { + Some(ch) if is_ident_continue(ch) => Err(Reject), + Some(_) | None => Ok(input), + } +} + +// Rustc's representation of a macro expansion error in expression position or +// type position. +const ERROR: &str = "(/*ERROR*/)"; + +pub(crate) fn token_stream(mut input: Cursor) -> Result { + let mut trees = TokenStreamBuilder::new(); + let mut stack = Vec::new(); + + loop { + input = skip_whitespace(input); + + if let Ok((rest, ())) = doc_comment(input, &mut trees) { + input = rest; + continue; + } + + #[cfg(span_locations)] + let lo = input.off; + + let first = match input.bytes().next() { + Some(first) => first, + None => match stack.last() { + None => return Ok(trees.build()), + #[cfg(span_locations)] + Some((lo, _frame)) => { + return Err(LexError { + span: Span { lo: *lo, hi: *lo }, + }) + } + #[cfg(not(span_locations))] + Some(_frame) => return Err(LexError { span: Span {} }), + }, + }; + + if let Some(open_delimiter) = match first { + b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis), + b'[' => Some(Delimiter::Bracket), + b'{' => Some(Delimiter::Brace), + _ => None, + } { + input = input.advance(1); + let frame = (open_delimiter, trees); + #[cfg(span_locations)] + let frame = (lo, frame); + stack.push(frame); + trees = TokenStreamBuilder::new(); + } else if let Some(close_delimiter) = match first { + b')' => Some(Delimiter::Parenthesis), + b']' => Some(Delimiter::Bracket), + b'}' => Some(Delimiter::Brace), + _ => None, + } { + let frame = match stack.pop() { + Some(frame) => frame, + None => return Err(lex_error(input)), + }; + #[cfg(span_locations)] + let (lo, frame) = frame; + let (open_delimiter, outer) = frame; + if open_delimiter != close_delimiter { + return Err(lex_error(input)); + } + input = input.advance(1); + let mut g = Group::new(open_delimiter, trees.build()); + g.set_span(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: input.off, + }); + trees = outer; + trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g))); + } else { + let (rest, mut tt) = match leaf_token(input) { + Ok((rest, tt)) => (rest, tt), + Err(Reject) => return Err(lex_error(input)), + }; + tt.set_span(crate::Span::_new_fallback(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + })); + trees.push_token_from_parser(tt); + input = rest; + } + } +} + +fn lex_error(cursor: Cursor) -> LexError { + #[cfg(not(span_locations))] + let _ = cursor; + LexError { + span: Span { + #[cfg(span_locations)] + lo: cursor.off, + #[cfg(span_locations)] + hi: cursor.off, + }, + } +} + +fn leaf_token(input: Cursor) -> PResult { + if let Ok((input, l)) = literal(input) { + // must be parsed before ident + Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l)))) + } else if let Ok((input, p)) = punct(input) { + Ok((input, TokenTree::Punct(p))) + } else if let Ok((input, i)) = ident(input) { + Ok((input, TokenTree::Ident(i))) + } else if input.starts_with(ERROR) { + let rest = input.advance(ERROR.len()); + let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned())); + Ok((rest, TokenTree::Literal(repr))) + } else { + Err(Reject) + } +} + +fn ident(input: Cursor) -> PResult { + if [ + "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#", + ] + .iter() + .any(|prefix| input.starts_with(prefix)) + { + Err(Reject) + } else { + ident_any(input) + } +} + +fn ident_any(input: Cursor) -> PResult { + let raw = input.starts_with("r#"); + let rest = input.advance((raw as usize) << 1); + + let (rest, sym) = ident_not_raw(rest)?; + + if !raw { + let ident = + crate::Ident::_new_fallback(Ident::new_unchecked(sym, fallback::Span::call_site())); + return Ok((rest, ident)); + } + + match sym { + "_" | "super" | "self" | "Self" | "crate" => return Err(Reject), + _ => {} + } + + let ident = + crate::Ident::_new_fallback(Ident::new_raw_unchecked(sym, fallback::Span::call_site())); + Ok((rest, ident)) +} + +fn ident_not_raw(input: Cursor) -> PResult<&str> { + let mut chars = input.char_indices(); + + match chars.next() { + Some((_, ch)) if is_ident_start(ch) => {} + _ => return Err(Reject), + } + + let mut end = input.len(); + for (i, ch) in chars { + if !is_ident_continue(ch) { + end = i; + break; + } + } + + Ok((input.advance(end), &input.rest[..end])) +} + +pub(crate) fn literal(input: Cursor) -> PResult { + let rest = literal_nocapture(input)?; + let end = input.len() - rest.len(); + Ok((rest, Literal::_new(input.rest[..end].to_string()))) +} + +fn literal_nocapture(input: Cursor) -> Result { + if let Ok(ok) = string(input) { + Ok(ok) + } else if let Ok(ok) = byte_string(input) { + Ok(ok) + } else if let Ok(ok) = c_string(input) { + Ok(ok) + } else if let Ok(ok) = byte(input) { + Ok(ok) + } else if let Ok(ok) = character(input) { + Ok(ok) + } else if let Ok(ok) = float(input) { + Ok(ok) + } else if let Ok(ok) = int(input) { + Ok(ok) + } else { + Err(Reject) + } +} + +fn literal_suffix(input: Cursor) -> Cursor { + match ident_not_raw(input) { + Ok((input, _)) => input, + Err(Reject) => input, + } +} + +fn string(input: Cursor) -> Result { + if let Ok(input) = input.parse("\"") { + cooked_string(input) + } else if let Ok(input) = input.parse("r") { + raw_string(input) + } else { + Err(Reject) + } +} + +fn cooked_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_char(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {} + Some((_, 'u')) => { + backslash_u(&mut chars)?; + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + _ch => {} + } + } + Err(Reject) +} + +fn raw_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + _ => {} + } + } + Err(Reject) +} + +fn byte_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("b\"") { + cooked_byte_string(input) + } else if let Ok(input) = input.parse("br") { + raw_byte_string(input) + } else { + Err(Reject) + } +} + +fn cooked_byte_string(mut input: Cursor) -> Result { + let mut bytes = input.bytes().enumerate(); + while let Some((offset, b)) = bytes.next() { + match b { + b'"' => { + let input = input.advance(offset + 1); + return Ok(literal_suffix(input)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\\' => match bytes.next() { + Some((_, b'x')) => { + backslash_x_byte(&mut bytes)?; + } + Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {} + Some((newline, b @ (b'\n' | b'\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, b)?; + bytes = input.bytes().enumerate(); + } + _ => break, + }, + b if b.is_ascii() => {} + _ => break, + } + } + Err(Reject) +} + +fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> { + for (i, byte) in input.bytes().enumerate() { + match byte { + b'"' => { + if i > 255 { + // https://github.com/rust-lang/rust/pull/95251 + return Err(Reject); + } + return Ok((input.advance(i + 1), &input.rest[..i])); + } + b'#' => {} + _ => break, + } + } + Err(Reject) +} + +fn raw_byte_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + other => { + if !other.is_ascii() { + break; + } + } + } + } + Err(Reject) +} + +fn c_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("c\"") { + cooked_c_string(input) + } else if let Ok(input) = input.parse("cr") { + raw_c_string(input) + } else { + Err(Reject) + } +} + +fn raw_c_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\0' => break, + _ => {} + } + } + Err(Reject) +} + +fn cooked_c_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_nonzero(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {} + Some((_, 'u')) => { + if backslash_u(&mut chars)? == '\0' { + break; + } + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + '\0' => break, + _ch => {} + } + } + Err(Reject) +} + +fn byte(input: Cursor) -> Result { + let input = input.parse("b'")?; + let mut bytes = input.bytes().enumerate(); + let ok = match bytes.next().map(|(_, b)| b) { + Some(b'\\') => match bytes.next().map(|(_, b)| b) { + Some(b'x') => backslash_x_byte(&mut bytes).is_ok(), + Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true, + _ => false, + }, + b => b.is_some(), + }; + if !ok { + return Err(Reject); + } + let (offset, _) = bytes.next().ok_or(Reject)?; + if !input.chars().as_str().is_char_boundary(offset) { + return Err(Reject); + } + let input = input.advance(offset).parse("'")?; + Ok(literal_suffix(input)) +} + +fn character(input: Cursor) -> Result { + let input = input.parse("'")?; + let mut chars = input.char_indices(); + let ok = match chars.next().map(|(_, ch)| ch) { + Some('\\') => match chars.next().map(|(_, ch)| ch) { + Some('x') => backslash_x_char(&mut chars).is_ok(), + Some('u') => backslash_u(&mut chars).is_ok(), + Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true, + _ => false, + }, + ch => ch.is_some(), + }; + if !ok { + return Err(Reject); + } + let (idx, _) = chars.next().ok_or(Reject)?; + let input = input.advance(idx).parse("'")?; + Ok(literal_suffix(input)) +} + +macro_rules! next_ch { + ($chars:ident @ $pat:pat) => { + match $chars.next() { + Some((_, ch)) => match ch { + $pat => ch, + _ => return Err(Reject), + }, + None => return Err(Reject), + } + }; +} + +fn backslash_x_char(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ '0'..='7'); + next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + Ok(()) +} + +fn backslash_x_byte(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + Ok(()) +} + +fn backslash_x_nonzero(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + if first == '0' && second == '0' { + Err(Reject) + } else { + Ok(()) + } +} + +fn backslash_u(chars: &mut I) -> Result +where + I: Iterator, +{ + next_ch!(chars @ '{'); + let mut value = 0; + let mut len = 0; + for (_, ch) in chars { + let digit = match ch { + '0'..='9' => ch as u8 - b'0', + 'a'..='f' => 10 + ch as u8 - b'a', + 'A'..='F' => 10 + ch as u8 - b'A', + '_' if len > 0 => continue, + '}' if len > 0 => return char::from_u32(value).ok_or(Reject), + _ => break, + }; + if len == 6 { + break; + } + value *= 0x10; + value += u32::from(digit); + len += 1; + } + Err(Reject) +} + +fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> { + let mut whitespace = input.bytes().enumerate(); + loop { + if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') { + return Err(Reject); + } + match whitespace.next() { + Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => { + last = b; + } + Some((offset, _)) => { + *input = input.advance(offset); + return Ok(()); + } + None => return Err(Reject), + } + } +} + +fn float(input: Cursor) -> Result { + let mut rest = float_digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn float_digits(input: Cursor) -> Result { + let mut chars = input.chars().peekable(); + match chars.next() { + Some(ch) if '0' <= ch && ch <= '9' => {} + _ => return Err(Reject), + } + + let mut len = 1; + let mut has_dot = false; + let mut has_exp = false; + while let Some(&ch) = chars.peek() { + match ch { + '0'..='9' | '_' => { + chars.next(); + len += 1; + } + '.' => { + if has_dot { + break; + } + chars.next(); + if chars + .peek() + .map_or(false, |&ch| ch == '.' || is_ident_start(ch)) + { + return Err(Reject); + } + len += 1; + has_dot = true; + } + 'e' | 'E' => { + chars.next(); + len += 1; + has_exp = true; + break; + } + _ => break, + } + } + + if !(has_dot || has_exp) { + return Err(Reject); + } + + if has_exp { + let token_before_exp = if has_dot { + Ok(input.advance(len - 1)) + } else { + Err(Reject) + }; + let mut has_sign = false; + let mut has_exp_value = false; + while let Some(&ch) = chars.peek() { + match ch { + '+' | '-' => { + if has_exp_value { + break; + } + if has_sign { + return token_before_exp; + } + chars.next(); + len += 1; + has_sign = true; + } + '0'..='9' => { + chars.next(); + len += 1; + has_exp_value = true; + } + '_' => { + chars.next(); + len += 1; + } + _ => break, + } + } + if !has_exp_value { + return token_before_exp; + } + } + + Ok(input.advance(len)) +} + +fn int(input: Cursor) -> Result { + let mut rest = digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn digits(mut input: Cursor) -> Result { + let base = if input.starts_with("0x") { + input = input.advance(2); + 16 + } else if input.starts_with("0o") { + input = input.advance(2); + 8 + } else if input.starts_with("0b") { + input = input.advance(2); + 2 + } else { + 10 + }; + + let mut len = 0; + let mut empty = true; + for b in input.bytes() { + match b { + b'0'..=b'9' => { + let digit = (b - b'0') as u64; + if digit >= base { + return Err(Reject); + } + } + b'a'..=b'f' => { + let digit = 10 + (b - b'a') as u64; + if digit >= base { + break; + } + } + b'A'..=b'F' => { + let digit = 10 + (b - b'A') as u64; + if digit >= base { + break; + } + } + b'_' => { + if empty && base == 10 { + return Err(Reject); + } + len += 1; + continue; + } + _ => break, + } + len += 1; + empty = false; + } + if empty { + Err(Reject) + } else { + Ok(input.advance(len)) + } +} + +fn punct(input: Cursor) -> PResult { + let (rest, ch) = punct_char(input)?; + if ch == '\'' { + let (after_lifetime, _ident) = ident_any(rest)?; + if after_lifetime.starts_with_char('\'') + || (after_lifetime.starts_with_char('#') && !rest.starts_with("r#")) + { + Err(Reject) + } else { + Ok((rest, Punct::new('\'', Spacing::Joint))) + } + } else { + let kind = match punct_char(rest) { + Ok(_) => Spacing::Joint, + Err(Reject) => Spacing::Alone, + }; + Ok((rest, Punct::new(ch, kind))) + } +} + +fn punct_char(input: Cursor) -> PResult { + if input.starts_with("//") || input.starts_with("/*") { + // Do not accept `/` of a comment as a punct. + return Err(Reject); + } + + let mut chars = input.chars(); + let first = match chars.next() { + Some(ch) => ch, + None => { + return Err(Reject); + } + }; + let recognized = "~!@#$%^&*-=+|;:,<.>/?'"; + if recognized.contains(first) { + Ok((input.advance(first.len_utf8()), first)) + } else { + Err(Reject) + } +} + +fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> { + #[cfg(span_locations)] + let lo = input.off; + let (rest, (comment, inner)) = doc_comment_contents(input)?; + let fallback_span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + let span = crate::Span::_new_fallback(fallback_span); + + let mut scan_for_bare_cr = comment; + while let Some(cr) = scan_for_bare_cr.find('\r') { + let rest = &scan_for_bare_cr[cr + 1..]; + if !rest.starts_with('\n') { + return Err(Reject); + } + scan_for_bare_cr = rest; + } + + let mut pound = Punct::new('#', Spacing::Alone); + pound.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(pound)); + + if inner { + let mut bang = Punct::new('!', Spacing::Alone); + bang.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(bang)); + } + + let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span)); + let mut equal = Punct::new('=', Spacing::Alone); + equal.set_span(span); + let mut literal = crate::Literal::_new_fallback(Literal::string(comment)); + literal.set_span(span); + let mut bracketed = TokenStreamBuilder::with_capacity(3); + bracketed.push_token_from_parser(TokenTree::Ident(doc_ident)); + bracketed.push_token_from_parser(TokenTree::Punct(equal)); + bracketed.push_token_from_parser(TokenTree::Literal(literal)); + let group = Group::new(Delimiter::Bracket, bracketed.build()); + let mut group = crate::Group::_new_fallback(group); + group.set_span(span); + trees.push_token_from_parser(TokenTree::Group(group)); + + Ok((rest, ())) +} + +fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> { + if input.starts_with("//!") { + let input = input.advance(3); + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, true))) + } else if input.starts_with("/*!") { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], true))) + } else if input.starts_with("///") { + let input = input.advance(3); + if input.starts_with_char('/') { + return Err(Reject); + } + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, false))) + } else if input.starts_with("/**") && !input.rest[3..].starts_with('*') { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], false))) + } else { + Err(Reject) + } +} + +fn take_until_newline_or_eof(input: Cursor) -> (Cursor, &str) { + let chars = input.char_indices(); + + for (i, ch) in chars { + if ch == '\n' { + return (input.advance(i), &input.rest[..i]); + } else if ch == '\r' && input.rest[i + 1..].starts_with('\n') { + return (input.advance(i + 1), &input.rest[..i]); + } + } + + (input.advance(input.len()), input.rest) +} diff --git a/rust/proc-macro2/probe.rs b/rust/proc-macro2/probe.rs new file mode 100644 index 000000000..d68aa8cfd --- /dev/null +++ b/rust/proc-macro2/probe.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(dead_code)] + +#[cfg(proc_macro_span)] +pub(crate) mod proc_macro_span; + +#[cfg(proc_macro_span_file)] +pub(crate) mod proc_macro_span_file; + +#[cfg(proc_macro_span_location)] +pub(crate) mod proc_macro_span_location; diff --git a/rust/proc-macro2/probe/proc_macro_span.rs b/rust/proc-macro2/probe/proc_macro_span.rs new file mode 100644 index 000000000..892a7eb3e --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// This code exercises the surface area that we expect of Span's unstable API. +// If the current toolchain is able to compile it, then proc-macro2 is able to +// offer these APIs too. + +#![cfg_attr(procmacro2_build_probe, feature(proc_macro_span))] + +extern crate proc_macro; + +use core::ops::{Range, RangeBounds}; +use proc_macro::{Literal, Span}; +use std::path::PathBuf; + +pub fn byte_range(this: &Span) -> Range { + this.byte_range() +} + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} + +pub fn join(this: &Span, other: Span) -> Option { + this.join(other) +} + +pub fn subspan>(this: &Literal, range: R) -> Option { + this.subspan(range) +} + +// Include in sccache cache key. +#[cfg(procmacro2_build_probe)] +const _: Option<&str> = option_env!("RUSTC_BOOTSTRAP"); diff --git a/rust/proc-macro2/probe/proc_macro_span_file.rs b/rust/proc-macro2/probe/proc_macro_span_file.rs new file mode 100644 index 000000000..f2dbc4056 --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_file.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; +use std::path::PathBuf; + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} diff --git a/rust/proc-macro2/probe/proc_macro_span_location.rs b/rust/proc-macro2/probe/proc_macro_span_location.rs new file mode 100644 index 000000000..ae19c9339 --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_location.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} diff --git a/rust/proc-macro2/rcvec.rs b/rust/proc-macro2/rcvec.rs new file mode 100644 index 000000000..e224ebe27 --- /dev/null +++ b/rust/proc-macro2/rcvec.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::rc::Rc; +use alloc::vec; +use core::mem; +use core::panic::RefUnwindSafe; +use core::slice; + +pub(crate) struct RcVec { + inner: Rc>, +} + +pub(crate) struct RcVecBuilder { + inner: Vec, +} + +pub(crate) struct RcVecMut<'a, T> { + inner: &'a mut Vec, +} + +#[derive(Clone)] +pub(crate) struct RcVecIntoIter { + inner: vec::IntoIter, +} + +impl RcVec { + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.inner.len() + } + + pub(crate) fn iter(&self) -> slice::Iter { + self.inner.iter() + } + + pub(crate) fn make_mut(&mut self) -> RcVecMut + where + T: Clone, + { + RcVecMut { + inner: Rc::make_mut(&mut self.inner), + } + } + + pub(crate) fn get_mut(&mut self) -> Option> { + let inner = Rc::get_mut(&mut self.inner)?; + Some(RcVecMut { inner }) + } + + pub(crate) fn make_owned(mut self) -> RcVecBuilder + where + T: Clone, + { + let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) { + mem::take(owned) + } else { + Vec::clone(&self.inner) + }; + RcVecBuilder { inner: vec } + } +} + +impl RcVecBuilder { + pub(crate) fn new() -> Self { + RcVecBuilder { inner: Vec::new() } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + RcVecBuilder { + inner: Vec::with_capacity(cap), + } + } + + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { + inner: &mut self.inner, + } + } + + pub(crate) fn build(self) -> RcVec { + RcVec { + inner: Rc::new(self.inner), + } + } +} + +impl<'a, T> RcVecMut<'a, T> { + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { inner: self.inner } + } + + pub(crate) fn take(self) -> RcVecBuilder { + let vec = mem::take(self.inner); + RcVecBuilder { inner: vec } + } +} + +impl Clone for RcVec { + fn clone(&self) -> Self { + RcVec { + inner: Rc::clone(&self.inner), + } + } +} + +impl IntoIterator for RcVecBuilder { + type Item = T; + type IntoIter = RcVecIntoIter; + + fn into_iter(self) -> Self::IntoIter { + RcVecIntoIter { + inner: self.inner.into_iter(), + } + } +} + +impl Iterator for RcVecIntoIter { + type Item = T; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl RefUnwindSafe for RcVec where T: RefUnwindSafe {} diff --git a/rust/proc-macro2/wrapper.rs b/rust/proc-macro2/wrapper.rs new file mode 100644 index 000000000..6792e9834 --- /dev/null +++ b/rust/proc-macro2/wrapper.rs @@ -0,0 +1,986 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::detection::inside_proc_macro; +use crate::fallback::{self, FromStr2 as _}; +#[cfg(span_locations)] +use crate::location::LineColumn; +#[cfg(proc_macro_span)] +use crate::probe::proc_macro_span; +#[cfg(all(span_locations, proc_macro_span_file))] +use crate::probe::proc_macro_span_file; +#[cfg(all(span_locations, proc_macro_span_location))] +use crate::probe::proc_macro_span_location; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::fmt::{self, Debug, Display}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[derive(Clone)] +pub(crate) enum TokenStream { + Compiler(DeferredTokenStream), + Fallback(fallback::TokenStream), +} + +// Work around https://github.com/rust-lang/rust/issues/65080. +// In `impl Extend for TokenStream` which is used heavily by quote, +// we hold on to the appended tokens and do proc_macro::TokenStream::extend as +// late as possible to batch together consecutive uses of the Extend impl. +#[derive(Clone)] +pub(crate) struct DeferredTokenStream { + stream: proc_macro::TokenStream, + extra: Vec, +} + +pub(crate) enum LexError { + Compiler(proc_macro::LexError), + Fallback(fallback::LexError), + + // Rustc was supposed to return a LexError, but it panicked instead. + // https://github.com/rust-lang/rust/issues/58736 + CompilerPanic, +} + +#[cold] +fn mismatch(line: u32) -> ! { + #[cfg(procmacro2_backtrace)] + { + let backtrace = std::backtrace::Backtrace::force_capture(); + panic!("compiler/fallback mismatch L{}\n\n{}", line, backtrace) + } + #[cfg(not(procmacro2_backtrace))] + { + panic!("compiler/fallback mismatch L{}", line) + } +} + +impl DeferredTokenStream { + fn new(stream: proc_macro::TokenStream) -> Self { + DeferredTokenStream { + stream, + extra: Vec::new(), + } + } + + fn is_empty(&self) -> bool { + self.stream.is_empty() && self.extra.is_empty() + } + + fn evaluate_now(&mut self) { + // If-check provides a fast short circuit for the common case of `extra` + // being empty, which saves a round trip over the proc macro bridge. + // Improves macro expansion time in winrt by 6% in debug mode. + if !self.extra.is_empty() { + self.stream.extend(self.extra.drain(..)); + } + } + + fn into_token_stream(mut self) -> proc_macro::TokenStream { + self.evaluate_now(); + self.stream + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new())) + } else { + TokenStream::Fallback(fallback::TokenStream::new()) + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + if inside_proc_macro() { + Ok(TokenStream::Compiler(DeferredTokenStream::new( + proc_macro::TokenStream::from_str_checked(src)?, + ))) + } else { + Ok(TokenStream::Fallback( + fallback::TokenStream::from_str_checked(src)?, + )) + } + } + + pub(crate) fn is_empty(&self) -> bool { + match self { + TokenStream::Compiler(tts) => tts.is_empty(), + TokenStream::Fallback(tts) => tts.is_empty(), + } + } + + fn unwrap_nightly(self) -> proc_macro::TokenStream { + match self { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + } + } + + fn unwrap_stable(self) -> fallback::TokenStream { + match self { + TokenStream::Compiler(_) => mismatch(line!()), + TokenStream::Fallback(s) => s, + } + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Display::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Display::fmt(tts, f), + } + } +} + +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::Compiler(DeferredTokenStream::new(inner)) + } +} + +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + match inner { + TokenStream::Compiler(inner) => inner.into_token_stream(), + TokenStream::Fallback(inner) => { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } + } + } +} + +impl From for TokenStream { + fn from(inner: fallback::TokenStream) -> Self { + TokenStream::Fallback(inner) + } +} + +// Assumes inside_proc_macro(). +fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree { + match token { + TokenTree::Group(tt) => proc_macro::TokenTree::Group(tt.inner.unwrap_nightly()), + TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + Spacing::Joint => proc_macro::Spacing::Joint, + Spacing::Alone => proc_macro::Spacing::Alone, + }; + let mut punct = proc_macro::Punct::new(tt.as_char(), spacing); + punct.set_span(tt.span().inner.unwrap_nightly()); + proc_macro::TokenTree::Punct(punct) + } + TokenTree::Ident(tt) => proc_macro::TokenTree::Ident(tt.inner.unwrap_nightly()), + TokenTree::Literal(tt) => proc_macro::TokenTree::Literal(tt.inner.unwrap_nightly()), + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::from( + into_compiler_token(token), + ))) + } else { + TokenStream::Fallback(fallback::TokenStream::from(token)) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new( + trees.into_iter().map(into_compiler_token).collect(), + )) + } else { + TokenStream::Fallback(trees.into_iter().collect()) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut streams = streams.into_iter(); + match streams.next() { + Some(TokenStream::Compiler(mut first)) => { + first.evaluate_now(); + first.stream.extend(streams.map(|s| match s { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + })); + TokenStream::Compiler(first) + } + Some(TokenStream::Fallback(mut first)) => { + first.extend(streams.map(|s| match s { + TokenStream::Fallback(s) => s, + TokenStream::Compiler(_) => mismatch(line!()), + })); + TokenStream::Fallback(first) + } + None => TokenStream::new(), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, stream: I) { + match self { + TokenStream::Compiler(tts) => { + // Here is the reason for DeferredTokenStream. + for token in stream { + tts.extra.push(into_compiler_token(token)); + } + } + TokenStream::Fallback(tts) => tts.extend(stream), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + match self { + TokenStream::Compiler(tts) => { + tts.evaluate_now(); + tts.stream + .extend(streams.into_iter().map(TokenStream::unwrap_nightly)); + } + TokenStream::Fallback(tts) => { + tts.extend(streams.into_iter().map(TokenStream::unwrap_stable)); + } + } + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Debug::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Debug::fmt(tts, f), + } + } +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + match self { + LexError::Compiler(_) | LexError::CompilerPanic => Span::call_site(), + LexError::Fallback(e) => Span::Fallback(e.span()), + } + } +} + +impl From for LexError { + fn from(e: proc_macro::LexError) -> Self { + LexError::Compiler(e) + } +} + +impl From for LexError { + fn from(e: fallback::LexError) -> Self { + LexError::Fallback(e) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Debug::fmt(e, f), + LexError::Fallback(e) => Debug::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Debug::fmt(&fallback, f) + } + } + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Display::fmt(e, f), + LexError::Fallback(e) => Display::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Display::fmt(&fallback, f) + } + } + } +} + +#[derive(Clone)] +pub(crate) enum TokenTreeIter { + Compiler(proc_macro::token_stream::IntoIter), + Fallback(fallback::TokenTreeIter), +} + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + match self { + TokenStream::Compiler(tts) => { + TokenTreeIter::Compiler(tts.into_token_stream().into_iter()) + } + TokenStream::Fallback(tts) => TokenTreeIter::Fallback(tts.into_iter()), + } + } +} + +impl Iterator for TokenTreeIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + let token = match self { + TokenTreeIter::Compiler(iter) => iter.next()?, + TokenTreeIter::Fallback(iter) => return iter.next(), + }; + Some(match token { + proc_macro::TokenTree::Group(tt) => { + TokenTree::Group(crate::Group::_new(Group::Compiler(tt))) + } + proc_macro::TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + proc_macro::Spacing::Joint => Spacing::Joint, + proc_macro::Spacing::Alone => Spacing::Alone, + }; + let mut o = Punct::new(tt.as_char(), spacing); + o.set_span(crate::Span::_new(Span::Compiler(tt.span()))); + TokenTree::Punct(o) + } + proc_macro::TokenTree::Ident(s) => { + TokenTree::Ident(crate::Ident::_new(Ident::Compiler(s))) + } + proc_macro::TokenTree::Literal(l) => { + TokenTree::Literal(crate::Literal::_new(Literal::Compiler(l))) + } + }) + } + + fn size_hint(&self) -> (usize, Option) { + match self { + TokenTreeIter::Compiler(tts) => tts.size_hint(), + TokenTreeIter::Fallback(tts) => tts.size_hint(), + } + } +} + +#[derive(Copy, Clone)] +pub(crate) enum Span { + Compiler(proc_macro::Span), + Fallback(fallback::Span), +} + +impl Span { + pub(crate) fn call_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::call_site()) + } else { + Span::Fallback(fallback::Span::call_site()) + } + } + + pub(crate) fn mixed_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::mixed_site()) + } else { + Span::Fallback(fallback::Span::mixed_site()) + } + } + + #[cfg(super_unstable)] + pub(crate) fn def_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::def_site()) + } else { + Span::Fallback(fallback::Span::def_site()) + } + } + + pub(crate) fn resolved_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn unwrap(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => panic!("proc_macro::Span is only available in procedural macros"), + } + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => proc_macro_span::byte_range(s), + #[cfg(not(proc_macro_span))] + Span::Compiler(_) => 0..0, + Span::Fallback(s) => s.byte_range(), + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => LineColumn { + line: proc_macro_span_location::line(s), + column: proc_macro_span_location::column(s).saturating_sub(1), + }, + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.start(), + } + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => { + let end = proc_macro_span_location::end(s); + LineColumn { + line: proc_macro_span_location::line(&end), + column: proc_macro_span_location::column(&end).saturating_sub(1), + } + } + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.end(), + } + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => "".to_owned(), + Span::Fallback(s) => s.file(), + } + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::local_file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => None, + Span::Fallback(s) => s.local_file(), + } + } + + pub(crate) fn join(&self, other: Span) -> Option { + let ret = match (self, other) { + #[cfg(proc_macro_span)] + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(proc_macro_span::join(a, b)?), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.join(b)?), + _ => return None, + }; + Some(ret) + } + + #[cfg(super_unstable)] + pub(crate) fn eq(&self, other: &Span) -> bool { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => a.eq(b), + (Span::Fallback(a), Span::Fallback(b)) => a.eq(b), + _ => false, + } + } + + pub(crate) fn source_text(&self) -> Option { + match self { + #[cfg(not(no_source_text))] + Span::Compiler(s) => s.source_text(), + #[cfg(no_source_text)] + Span::Compiler(_) => None, + Span::Fallback(s) => s.source_text(), + } + } + + fn unwrap_nightly(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for crate::Span { + fn from(proc_span: proc_macro::Span) -> Self { + crate::Span::_new(Span::Compiler(proc_span)) + } +} + +impl From for Span { + fn from(inner: fallback::Span) -> Self { + Span::Fallback(inner) + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Span::Compiler(s) => Debug::fmt(s, f), + Span::Fallback(s) => Debug::fmt(s, f), + } + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + match span { + Span::Compiler(s) => { + debug.field("span", &s); + } + Span::Fallback(s) => fallback::debug_span_field_if_nontrivial(debug, s), + } +} + +#[derive(Clone)] +pub(crate) enum Group { + Compiler(proc_macro::Group), + Fallback(fallback::Group), +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + match stream { + TokenStream::Compiler(tts) => { + let delimiter = match delimiter { + Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis, + Delimiter::Bracket => proc_macro::Delimiter::Bracket, + Delimiter::Brace => proc_macro::Delimiter::Brace, + Delimiter::None => proc_macro::Delimiter::None, + }; + Group::Compiler(proc_macro::Group::new(delimiter, tts.into_token_stream())) + } + TokenStream::Fallback(stream) => { + Group::Fallback(fallback::Group::new(delimiter, stream)) + } + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + match self { + Group::Compiler(g) => match g.delimiter() { + proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, + proc_macro::Delimiter::Bracket => Delimiter::Bracket, + proc_macro::Delimiter::Brace => Delimiter::Brace, + proc_macro::Delimiter::None => Delimiter::None, + }, + Group::Fallback(g) => g.delimiter(), + } + } + + pub(crate) fn stream(&self) -> TokenStream { + match self { + Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())), + Group::Fallback(g) => TokenStream::Fallback(g.stream()), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span()), + Group::Fallback(g) => Span::Fallback(g.span()), + } + } + + pub(crate) fn span_open(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_open()), + Group::Fallback(g) => Span::Fallback(g.span_open()), + } + } + + pub(crate) fn span_close(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_close()), + Group::Fallback(g) => Span::Fallback(g.span_close()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Group::Compiler(g), Span::Compiler(s)) => g.set_span(s), + (Group::Fallback(g), Span::Fallback(s)) => g.set_span(s), + (Group::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Group::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Group { + match self { + Group::Compiler(g) => g, + Group::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Group { + fn from(g: fallback::Group) -> Self { + Group::Fallback(g) + } +} + +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Display::fmt(group, formatter), + Group::Fallback(group) => Display::fmt(group, formatter), + } + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Debug::fmt(group, formatter), + Group::Fallback(group) => Debug::fmt(group, formatter), + } + } +} + +#[derive(Clone)] +pub(crate) enum Ident { + Compiler(proc_macro::Ident), + Fallback(fallback::Ident), +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_checked(string, s)), + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw_checked(string, s)), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Ident::Compiler(t) => Span::Compiler(t.span()), + Ident::Fallback(t) => Span::Fallback(t.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Ident::Compiler(t), Span::Compiler(s)) => t.set_span(s), + (Ident::Fallback(t), Span::Fallback(s)) => t.set_span(s), + (Ident::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Ident { + match self { + Ident::Compiler(s) => s, + Ident::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Ident { + fn from(inner: fallback::Ident) -> Self { + Ident::Fallback(inner) + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + match (self, other) { + (Ident::Compiler(t), Ident::Compiler(o)) => t.to_string() == o.to_string(), + (Ident::Fallback(t), Ident::Fallback(o)) => t == o, + (Ident::Compiler(_), Ident::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Ident::Compiler(_)) => mismatch(line!()), + } + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + match self { + Ident::Compiler(t) => t.to_string() == other, + Ident::Fallback(t) => t == other, + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Display::fmt(t, f), + Ident::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Debug::fmt(t, f), + Ident::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[derive(Clone)] +pub(crate) enum Literal { + Compiler(proc_macro::Literal), + Fallback(fallback::Literal), +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +macro_rules! unsuffixed_integers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +impl Literal { + pub(crate) fn from_str_checked(repr: &str) -> Result { + if inside_proc_macro() { + let literal = proc_macro::Literal::from_str_checked(repr)?; + Ok(Literal::Compiler(literal)) + } else { + let literal = fallback::Literal::from_str_checked(repr)?; + Ok(Literal::Fallback(literal)) + } + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::from_str_unchecked(repr)) + } else { + Literal::Fallback(unsafe { fallback::Literal::from_str_unchecked(repr) }) + } + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_integers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f32_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f32_unsuffixed(f)) + } + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f64_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f64_unsuffixed(f)) + } + } + + pub(crate) fn string(string: &str) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::string(string)) + } else { + Literal::Fallback(fallback::Literal::string(string)) + } + } + + pub(crate) fn character(ch: char) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::character(ch)) + } else { + Literal::Fallback(fallback::Literal::character(ch)) + } + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_byte_character))] + { + proc_macro::Literal::byte_character(byte) + } + + #[cfg(no_literal_byte_character)] + { + let fallback = fallback::Literal::byte_character(byte); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::byte_character(byte)) + } + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::byte_string(bytes)) + } else { + Literal::Fallback(fallback::Literal::byte_string(bytes)) + } + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_c_string))] + { + proc_macro::Literal::c_string(string) + } + + #[cfg(no_literal_c_string)] + { + let fallback = fallback::Literal::c_string(string); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::c_string(string)) + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Literal::Compiler(lit) => Span::Compiler(lit.span()), + Literal::Fallback(lit) => Span::Fallback(lit.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Literal::Compiler(lit), Span::Compiler(s)) => lit.set_span(s), + (Literal::Fallback(lit), Span::Fallback(s)) => lit.set_span(s), + (Literal::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Literal::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + match self { + #[cfg(proc_macro_span)] + Literal::Compiler(lit) => proc_macro_span::subspan(lit, range).map(Span::Compiler), + #[cfg(not(proc_macro_span))] + Literal::Compiler(_lit) => None, + Literal::Fallback(lit) => lit.subspan(range).map(Span::Fallback), + } + } + + fn unwrap_nightly(self) -> proc_macro::Literal { + match self { + Literal::Compiler(s) => s, + Literal::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Literal { + fn from(s: fallback::Literal) -> Self { + Literal::Fallback(s) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Display::fmt(t, f), + Literal::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Debug::fmt(t, f), + Literal::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + if inside_proc_macro() { + panic!( + "proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros" + ); + } else { + crate::fallback::invalidate_current_thread_spans(); + } +} diff --git a/rust/quote/README.md b/rust/quote/README.md new file mode 100644 index 000000000..186123eff --- /dev/null +++ b/rust/quote/README.md @@ -0,0 +1,12 @@ +# `quote` + +These source files come from the Rust `quote` crate, version 1.0.40 +(released 2025-03-12), hosted in the +repository, licensed under "Apache-2.0 OR MIT" and only modified to add +the SPDX license identifiers. + +For copyright details, please see: + + https://github.com/dtolnay/quote/blob/1.0.40/README.md#license + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-APACHE + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-MIT diff --git a/rust/quote/ext.rs b/rust/quote/ext.rs new file mode 100644 index 000000000..977d2f0c5 --- /dev/null +++ b/rust/quote/ext.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use super::ToTokens; +use core::iter; +use proc_macro2::{TokenStream, TokenTree}; + +/// TokenStream extension trait with methods for appending tokens. +/// +/// This trait is sealed and cannot be implemented outside of the `quote` crate. +pub trait TokenStreamExt: private::Sealed { + /// For use by `ToTokens` implementations. + /// + /// Appends the token specified to this list of tokens. + fn append(&mut self, token: U) + where + U: Into; + + /// For use by `ToTokens` implementations. + /// + /// ``` + /// # use quote::{quote, TokenStreamExt, ToTokens}; + /// # use proc_macro2::TokenStream; + /// # + /// struct X; + /// + /// impl ToTokens for X { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// tokens.append_all(&[true, false]); + /// } + /// } + /// + /// let tokens = quote!(#X); + /// assert_eq!(tokens.to_string(), "true false"); + /// ``` + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all of the items in the iterator `I`, separated by the tokens + /// `U`. + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all tokens in the iterator `I`, appending `U` after each + /// element, including after the last element of the iterator. + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; +} + +impl TokenStreamExt for TokenStream { + fn append(&mut self, token: U) + where + U: Into, + { + self.extend(iter::once(token.into())); + } + + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens, + { + for token in iter { + token.to_tokens(self); + } + } + + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for (i, token) in iter.into_iter().enumerate() { + if i > 0 { + op.to_tokens(self); + } + token.to_tokens(self); + } + } + + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for token in iter { + token.to_tokens(self); + term.to_tokens(self); + } + } +} + +mod private { + use proc_macro2::TokenStream; + + pub trait Sealed {} + + impl Sealed for TokenStream {} +} diff --git a/rust/quote/format.rs b/rust/quote/format.rs new file mode 100644 index 000000000..6e3d55b6e --- /dev/null +++ b/rust/quote/format.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Formatting macro for constructing `Ident`s. +/// +///
+/// +/// # Syntax +/// +/// Syntax is copied from the [`format!`] macro, supporting both positional and +/// named arguments. +/// +/// Only a limited set of formatting traits are supported. The current mapping +/// of format types to traits is: +/// +/// * `{}` ⇒ [`IdentFragment`] +/// * `{:o}` ⇒ [`Octal`](std::fmt::Octal) +/// * `{:x}` ⇒ [`LowerHex`](std::fmt::LowerHex) +/// * `{:X}` ⇒ [`UpperHex`](std::fmt::UpperHex) +/// * `{:b}` ⇒ [`Binary`](std::fmt::Binary) +/// +/// See [`std::fmt`] for more information. +/// +///
+/// +/// # IdentFragment +/// +/// Unlike `format!`, this macro uses the [`IdentFragment`] formatting trait by +/// default. This trait is like `Display`, with a few differences: +/// +/// * `IdentFragment` is only implemented for a limited set of types, such as +/// unsigned integers and strings. +/// * [`Ident`] arguments will have their `r#` prefixes stripped, if present. +/// +/// [`IdentFragment`]: crate::IdentFragment +/// [`Ident`]: proc_macro2::Ident +/// +///
+/// +/// # Hygiene +/// +/// The [`Span`] of the first `Ident` argument is used as the span of the final +/// identifier, falling back to [`Span::call_site`] when no identifiers are +/// provided. +/// +/// ``` +/// # use quote::format_ident; +/// # let ident = format_ident!("Ident"); +/// // If `ident` is an Ident, the span of `my_ident` will be inherited from it. +/// let my_ident = format_ident!("My{}{}", ident, "IsCool"); +/// assert_eq!(my_ident, "MyIdentIsCool"); +/// ``` +/// +/// Alternatively, the span can be overridden by passing the `span` named +/// argument. +/// +/// ``` +/// # use quote::format_ident; +/// # const IGNORE_TOKENS: &'static str = stringify! { +/// let my_span = /* ... */; +/// # }; +/// # let my_span = proc_macro2::Span::call_site(); +/// format_ident!("MyIdent", span = my_span); +/// ``` +/// +/// [`Span`]: proc_macro2::Span +/// [`Span::call_site`]: proc_macro2::Span::call_site +/// +///


+/// +/// # Panics +/// +/// This method will panic if the resulting formatted string is not a valid +/// identifier. +/// +///
+/// +/// # Examples +/// +/// Composing raw and non-raw identifiers: +/// ``` +/// # use quote::format_ident; +/// let my_ident = format_ident!("My{}", "Ident"); +/// assert_eq!(my_ident, "MyIdent"); +/// +/// let raw = format_ident!("r#Raw"); +/// assert_eq!(raw, "r#Raw"); +/// +/// let my_ident_raw = format_ident!("{}Is{}", my_ident, raw); +/// assert_eq!(my_ident_raw, "MyIdentIsRaw"); +/// ``` +/// +/// Integer formatting options: +/// ``` +/// # use quote::format_ident; +/// let num: u32 = 10; +/// +/// let decimal = format_ident!("Id_{}", num); +/// assert_eq!(decimal, "Id_10"); +/// +/// let octal = format_ident!("Id_{:o}", num); +/// assert_eq!(octal, "Id_12"); +/// +/// let binary = format_ident!("Id_{:b}", num); +/// assert_eq!(binary, "Id_1010"); +/// +/// let lower_hex = format_ident!("Id_{:x}", num); +/// assert_eq!(lower_hex, "Id_a"); +/// +/// let upper_hex = format_ident!("Id_{:X}", num); +/// assert_eq!(upper_hex, "Id_A"); +/// ``` +#[macro_export] +macro_rules! format_ident { + ($fmt:expr) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ]) + }; + + ($fmt:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ] $($rest)*) + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! format_ident_impl { + // Final state + ([$span:expr, $($fmt:tt)*]) => { + $crate::__private::mk_ident( + &$crate::__private::format!($($fmt)*), + $span, + ) + }; + + // Span argument + ([$old:expr, $($fmt:tt)*] span = $span:expr) => { + $crate::format_ident_impl!([$old, $($fmt)*] span = $span,) + }; + ([$old:expr, $($fmt:tt)*] span = $span:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::Some::<$crate::__private::Span>($span), + $($fmt)* + ] $($rest)*) + }; + + // Named argument + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $name = $arg,) + }; + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, $name = arg] $($rest)*), + } + }; + + // Positional argument + ([$span:expr, $($fmt:tt)*] $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $arg,) + }; + ([$span:expr, $($fmt:tt)*] $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, arg] $($rest)*), + } + }; +} diff --git a/rust/quote/ident_fragment.rs b/rust/quote/ident_fragment.rs new file mode 100644 index 000000000..d98106f17 --- /dev/null +++ b/rust/quote/ident_fragment.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use alloc::borrow::Cow; +use core::fmt; +use proc_macro2::{Ident, Span}; + +/// Specialized formatting trait used by `format_ident!`. +/// +/// [`Ident`] arguments formatted using this trait will have their `r#` prefix +/// stripped, if present. +/// +/// See [`format_ident!`] for more information. +/// +/// [`format_ident!`]: crate::format_ident +pub trait IdentFragment { + /// Format this value as an identifier fragment. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result; + + /// Span associated with this `IdentFragment`. + /// + /// If non-`None`, may be inherited by formatted identifiers. + fn span(&self) -> Option { + None + } +} + +impl IdentFragment for &T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for &mut T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for Ident { + fn span(&self) -> Option { + Some(self.span()) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let id = self.to_string(); + if let Some(id) = id.strip_prefix("r#") { + fmt::Display::fmt(id, f) + } else { + fmt::Display::fmt(&id[..], f) + } + } +} + +impl IdentFragment for Cow<'_, T> +where + T: IdentFragment + ToOwned + ?Sized, +{ + fn span(&self) -> Option { + T::span(self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + T::fmt(self, f) + } +} + +// Limited set of types which this is implemented for, as we want to avoid types +// which will often include non-identifier characters in their `Display` impl. +macro_rules! ident_fragment_display { + ($($T:ty),*) => { + $( + impl IdentFragment for $T { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self, f) + } + } + )* + }; +} + +ident_fragment_display!(bool, str, String, char); +ident_fragment_display!(u8, u16, u32, u64, u128, usize); diff --git a/rust/quote/lib.rs b/rust/quote/lib.rs new file mode 100644 index 000000000..cc1637660 --- /dev/null +++ b/rust/quote/lib.rs @@ -0,0 +1,1456 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! [![github]](https://github.com/dtolnay/quote) [![crates-io]](https://crates.io/crates/quote) [![docs-rs]](https://docs.rs/quote) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! This crate provides the [`quote!`] macro for turning Rust syntax tree data +//! structures into tokens of source code. +//! +//! Procedural macros in Rust receive a stream of tokens as input, execute +//! arbitrary Rust code to determine how to manipulate those tokens, and produce +//! a stream of tokens to hand back to the compiler to compile into the caller's +//! crate. Quasi-quoting is a solution to one piece of that — producing +//! tokens to return to the compiler. +//! +//! The idea of quasi-quoting is that we write *code* that we treat as *data*. +//! Within the `quote!` macro, we can write what looks like code to our text +//! editor or IDE. We get all the benefits of the editor's brace matching, +//! syntax highlighting, indentation, and maybe autocompletion. But rather than +//! compiling that as code into the current crate, we can treat it as data, pass +//! it around, mutate it, and eventually hand it back to the compiler as tokens +//! to compile into the macro caller's crate. +//! +//! This crate is motivated by the procedural macro use case, but is a +//! general-purpose Rust quasi-quoting library and is not specific to procedural +//! macros. +//! +//! ```toml +//! [dependencies] +//! quote = "1.0" +//! ``` +//! +//!
+//! +//! # Example +//! +//! The following quasi-quoted block of code is something you might find in [a] +//! procedural macro having to do with data structure serialization. The `#var` +//! syntax performs interpolation of runtime variables into the quoted tokens. +//! Check out the documentation of the [`quote!`] macro for more detail about +//! the syntax. See also the [`quote_spanned!`] macro which is important for +//! implementing hygienic procedural macros. +//! +//! [a]: https://serde.rs/ +//! +//! ``` +//! # use quote::quote; +//! # +//! # let generics = ""; +//! # let where_clause = ""; +//! # let field_ty = ""; +//! # let item_ty = ""; +//! # let path = ""; +//! # let value = ""; +//! # +//! let tokens = quote! { +//! struct SerializeWith #generics #where_clause { +//! value: &'a #field_ty, +//! phantom: core::marker::PhantomData<#item_ty>, +//! } +//! +//! impl #generics serde::Serialize for SerializeWith #generics #where_clause { +//! fn serialize(&self, serializer: S) -> Result +//! where +//! S: serde::Serializer, +//! { +//! #path(self.value, serializer) +//! } +//! } +//! +//! SerializeWith { +//! value: #value, +//! phantom: core::marker::PhantomData::<#item_ty>, +//! } +//! }; +//! ``` +//! +//!
+//! +//! # Non-macro code generators +//! +//! When using `quote` in a build.rs or main.rs and writing the output out to a +//! file, consider having the code generator pass the tokens through +//! [prettyplease] before writing. This way if an error occurs in the generated +//! code it is convenient for a human to read and debug. +//! +//! [prettyplease]: https://github.com/dtolnay/prettyplease + +// Quote types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/quote/1.0.40")] +#![allow( + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::missing_errors_doc, + clippy::missing_panics_doc, + clippy::module_name_repetitions, + clippy::needless_lifetimes, + // false positive https://github.com/rust-lang/rust-clippy/issues/6983 + clippy::wrong_self_convention, +)] + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod ext; +mod format; +mod ident_fragment; +mod to_tokens; + +// Not public API. +#[doc(hidden)] +#[path = "runtime.rs"] +pub mod __private; + +pub use crate::ext::TokenStreamExt; +pub use crate::ident_fragment::IdentFragment; +pub use crate::to_tokens::ToTokens; + +// Not public API. +#[doc(hidden)] +pub mod spanned; + +macro_rules! __quote { + ($quote:item) => { + /// The whole point. + /// + /// Performs variable interpolation against the input and produces it as + /// [`proc_macro2::TokenStream`]. + /// + /// Note: for returning tokens to the compiler in a procedural macro, use + /// `.into()` on the result to convert to [`proc_macro::TokenStream`]. + /// + ///
+ /// + /// # Interpolation + /// + /// Variable interpolation is done with `#var` (similar to `$var` in + /// `macro_rules!` macros). This grabs the `var` variable that is currently in + /// scope and inserts it in that location in the output tokens. Any type + /// implementing the [`ToTokens`] trait can be interpolated. This includes most + /// Rust primitive types as well as most of the syntax tree types from the [Syn] + /// crate. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// Repetition is done using `#(...)*` or `#(...),*` again similar to + /// `macro_rules!`. This iterates through the elements of any variable + /// interpolated within the repetition and inserts a copy of the repetition body + /// for each one. The variables in an interpolation may be a `Vec`, slice, + /// `BTreeSet`, or any `Iterator`. + /// + /// - `#(#var)*` — no separators + /// - `#(#var),*` — the character before the asterisk is used as a separator + /// - `#( struct #var; )*` — the repetition can contain other tokens + /// - `#( #k => println!("{}", #v), )*` — even multiple interpolations + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote!` + /// invocation are spanned with [`Span::call_site()`]. + /// + /// [`Span::call_site()`]: proc_macro2::Span::call_site + /// + /// A different span can be provided through the [`quote_spanned!`] macro. + /// + ///
+ /// + /// # Return type + /// + /// The macro evaluates to an expression of type `proc_macro2::TokenStream`. + /// Meanwhile Rust procedural macros are expected to return the type + /// `proc_macro::TokenStream`. + /// + /// The difference between the two types is that `proc_macro` types are entirely + /// specific to procedural macros and cannot ever exist in code outside of a + /// procedural macro, while `proc_macro2` types may exist anywhere including + /// tests and non-macro code like main.rs and build.rs. This is why even the + /// procedural macro ecosystem is largely built around `proc_macro2`, because + /// that ensures the libraries are unit testable and accessible in non-macro + /// contexts. + /// + /// There is a [`From`]-conversion in both directions so returning the output of + /// `quote!` from a procedural macro usually looks like `tokens.into()` or + /// `proc_macro::TokenStream::from(tokens)`. + /// + ///
+ /// + /// # Examples + /// + /// ### Procedural macro + /// + /// The structure of a basic procedural macro is as follows. Refer to the [Syn] + /// crate for further useful guidance on using `quote!` as part of a procedural + /// macro. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// ``` + /// # #[cfg(any())] + /// extern crate proc_macro; + /// # extern crate proc_macro2; + /// + /// # #[cfg(any())] + /// use proc_macro::TokenStream; + /// # use proc_macro2::TokenStream; + /// use quote::quote; + /// + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// #[proc_macro_derive(HeapSize)] + /// # }; + /// pub fn derive_heap_size(input: TokenStream) -> TokenStream { + /// // Parse the input and figure out what implementation to generate... + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let name = /* ... */; + /// let expr = /* ... */; + /// # }; + /// # + /// # let name = 0; + /// # let expr = 0; + /// + /// let expanded = quote! { + /// // The generated impl. + /// impl heapsize::HeapSize for #name { + /// fn heap_size_of_children(&self) -> usize { + /// #expr + /// } + /// } + /// }; + /// + /// // Hand the output tokens back to the compiler. + /// TokenStream::from(expanded) + /// } + /// ``` + /// + ///


+ /// + /// ### Combining quoted fragments + /// + /// Usually you don't end up constructing an entire final `TokenStream` in one + /// piece. Different parts may come from different helper functions. The tokens + /// produced by `quote!` themselves implement `ToTokens` and so can be + /// interpolated into later `quote!` invocations to build up a final result. + /// + /// ``` + /// # use quote::quote; + /// # + /// let type_definition = quote! {...}; + /// let methods = quote! {...}; + /// + /// let tokens = quote! { + /// #type_definition + /// #methods + /// }; + /// ``` + /// + ///


+ /// + /// ### Constructing identifiers + /// + /// Suppose we have an identifier `ident` which came from somewhere in a macro + /// input and we need to modify it in some way for the macro output. Let's + /// consider prepending the identifier with an underscore. + /// + /// Simply interpolating the identifier next to an underscore will not have the + /// behavior of concatenating them. The underscore and the identifier will + /// continue to be two separate tokens as if you had written `_ x`. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// // incorrect + /// quote! { + /// let mut _#ident = 0; + /// } + /// # ; + /// ``` + /// + /// The solution is to build a new identifier token with the correct value. As + /// this is such a common case, the [`format_ident!`] macro provides a + /// convenient utility for doing so correctly. + /// + /// ``` + /// # use proc_macro2::{Ident, Span}; + /// # use quote::{format_ident, quote}; + /// # + /// # let ident = Ident::new("i", Span::call_site()); + /// # + /// let varname = format_ident!("_{}", ident); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + /// Alternatively, the APIs provided by Syn and proc-macro2 can be used to + /// directly build the identifier. This is roughly equivalent to the above, but + /// will not handle `ident` being a raw identifier. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// let concatenated = format!("_{}", ident); + /// let varname = syn::Ident::new(&concatenated, ident.span()); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Making method calls + /// + /// Let's say our macro requires some type specified in the macro input to have + /// a constructor called `new`. We have the type in a variable called + /// `field_type` of type `syn::Type` and want to invoke the constructor. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// // incorrect + /// quote! { + /// let value = #field_type::new(); + /// } + /// # ; + /// ``` + /// + /// This works only sometimes. If `field_type` is `String`, the expanded code + /// contains `String::new()` which is fine. But if `field_type` is something + /// like `Vec` then the expanded code is `Vec::new()` which is invalid + /// syntax. Ordinarily in handwritten Rust we would write `Vec::::new()` + /// but for macros often the following is more convenient. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type>::new(); + /// } + /// # ; + /// ``` + /// + /// This expands to `>::new()` which behaves correctly. + /// + /// A similar pattern is appropriate for trait methods. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type as core::default::Default>::default(); + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Interpolating text inside of doc comments + /// + /// Neither doc comments nor string literals get interpolation behavior in + /// quote: + /// + /// ```compile_fail + /// quote! { + /// /// try to interpolate: #ident + /// /// + /// /// ... + /// } + /// ``` + /// + /// ```compile_fail + /// quote! { + /// #[doc = "try to interpolate: #ident"] + /// } + /// ``` + /// + /// Instead the best way to build doc comments that involve variables is by + /// formatting the doc string literal outside of quote. + /// + /// ```rust + /// # use proc_macro2::{Ident, Span}; + /// # use quote::quote; + /// # + /// # const IGNORE: &str = stringify! { + /// let msg = format!(...); + /// # }; + /// # + /// # let ident = Ident::new("var", Span::call_site()); + /// # let msg = format!("try to interpolate: {}", ident); + /// quote! { + /// #[doc = #msg] + /// /// + /// /// ... + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Indexing into a tuple struct + /// + /// When interpolating indices of a tuple or tuple struct, we need them not to + /// appears suffixed as integer literals by interpolating them as [`syn::Index`] + /// instead. + /// + /// [`syn::Index`]: https://docs.rs/syn/2.0/syn/struct.Index.html + /// + /// ```compile_fail + /// let i = 0usize..self.fields.len(); + /// + /// // expands to 0 + self.0usize.heap_size() + self.1usize.heap_size() + ... + /// // which is not valid syntax + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// ``` + /// + /// ``` + /// # use proc_macro2::{Ident, TokenStream}; + /// # use quote::quote; + /// # + /// # mod syn { + /// # use proc_macro2::{Literal, TokenStream}; + /// # use quote::{ToTokens, TokenStreamExt}; + /// # + /// # pub struct Index(usize); + /// # + /// # impl From for Index { + /// # fn from(i: usize) -> Self { + /// # Index(i) + /// # } + /// # } + /// # + /// # impl ToTokens for Index { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # tokens.append(Literal::usize_unsuffixed(self.0)); + /// # } + /// # } + /// # } + /// # + /// # struct Struct { + /// # fields: Vec, + /// # } + /// # + /// # impl Struct { + /// # fn example(&self) -> TokenStream { + /// let i = (0..self.fields.len()).map(syn::Index::from); + /// + /// // expands to 0 + self.0.heap_size() + self.1.heap_size() + ... + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// # } + /// # } + /// ``` + $quote + }; +} + +#[cfg(doc)] +__quote![ + #[macro_export] + macro_rules! quote { + ($($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote![ + #[macro_export] + macro_rules! quote { + () => { + $crate::__private::TokenStream::new() + }; + + // Special case rule for a single tt, for performance. + ($tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt _s} + _s + }}; + + // Special case rules for two tts, for performance. + (# $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt1 _s} + $crate::quote_token!{$tt2 _s} + _s + }}; + + // Rule for any other number of tokens. + ($($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_each_token!{_s $($tt)*} + _s + }}; + } +]; + +macro_rules! __quote_spanned { + ($quote_spanned:item) => { + /// Same as `quote!`, but applies a given span to all tokens originating within + /// the macro invocation. + /// + ///
+ /// + /// # Syntax + /// + /// A span expression of type [`Span`], followed by `=>`, followed by the tokens + /// to quote. The span expression should be brief — use a variable for + /// anything more than a few characters. There should be no space before the + /// `=>` token. + /// + /// [`Span`]: proc_macro2::Span + /// + /// ``` + /// # use proc_macro2::Span; + /// # use quote::quote_spanned; + /// # + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let span = /* ... */; + /// # }; + /// # let span = Span::call_site(); + /// # let init = 0; + /// + /// // On one line, use parentheses. + /// let tokens = quote_spanned!(span=> Box::into_raw(Box::new(#init))); + /// + /// // On multiple lines, place the span at the top and use braces. + /// let tokens = quote_spanned! {span=> + /// Box::into_raw(Box::new(#init)) + /// }; + /// ``` + /// + /// The lack of space before the `=>` should look jarring to Rust programmers + /// and this is intentional. The formatting is designed to be visibly + /// off-balance and draw the eye a particular way, due to the span expression + /// being evaluated in the context of the procedural macro and the remaining + /// tokens being evaluated in the generated code. + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote_spanned!` + /// invocation are spanned with the given span argument. + /// + ///
+ /// + /// # Example + /// + /// The following procedural macro code uses `quote_spanned!` to assert that a + /// particular Rust type implements the [`Sync`] trait so that references can be + /// safely shared between threads. + /// + /// ``` + /// # use quote::{quote_spanned, TokenStreamExt, ToTokens}; + /// # use proc_macro2::{Span, TokenStream}; + /// # + /// # struct Type; + /// # + /// # impl Type { + /// # fn span(&self) -> Span { + /// # Span::call_site() + /// # } + /// # } + /// # + /// # impl ToTokens for Type { + /// # fn to_tokens(&self, _tokens: &mut TokenStream) {} + /// # } + /// # + /// # let ty = Type; + /// # let call_site = Span::call_site(); + /// # + /// let ty_span = ty.span(); + /// let assert_sync = quote_spanned! {ty_span=> + /// struct _AssertSync where #ty: Sync; + /// }; + /// ``` + /// + /// If the assertion fails, the user will see an error like the following. The + /// input span of their type is highlighted in the error. + /// + /// ```text + /// error[E0277]: the trait bound `*const (): std::marker::Sync` is not satisfied + /// --> src/main.rs:10:21 + /// | + /// 10 | static ref PTR: *const () = &(); + /// | ^^^^^^^^^ `*const ()` cannot be shared between threads safely + /// ``` + /// + /// In this example it is important for the where-clause to be spanned with the + /// line/column information of the user's input type so that error messages are + /// placed appropriately by the compiler. + $quote_spanned + }; +} + +#[cfg(doc)] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=> $($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=>) => {{ + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::__private::TokenStream::new() + }}; + + // Special case rule for a single tt, for performance. + ($span:expr=> $tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt _s _span} + _s + }}; + + // Special case rules for two tts, for performance. + ($span:expr=> # $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($span:expr=> $tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt1 _s _span} + $crate::quote_token_spanned!{$tt2 _s _span} + _s + }}; + + // Rule for any other number of tokens. + ($span:expr=> $($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_each_token_spanned!{_s _span $($tt)*} + _s + }}; + } +]; + +// Extract the names of all #metavariables and pass them to the $call macro. +// +// in: pounded_var_names!(then!(...) a #b c #( #d )* #e) +// out: then!(... b); +// then!(... d); +// then!(... e); +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names { + ($call:ident! $extra:tt $($tts:tt)*) => { + $crate::pounded_var_names_with_context!{$call! $extra + (@ $($tts)*) + ($($tts)* @) + } + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names_with_context { + ($call:ident! $extra:tt ($($b1:tt)*) ($($curr:tt)*)) => { + $( + $crate::pounded_var_with_context!{$call! $extra $b1 $curr} + )* + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_with_context { + ($call:ident! $extra:tt $b1:tt ( $($inner:tt)* )) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt [ $($inner:tt)* ]) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt { $($inner:tt)* }) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident!($($extra:tt)*) # $var:ident) => { + $crate::$call!($($extra)* $var); + }; + + ($call:ident! $extra:tt $b1:tt $curr:tt) => {}; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_into_iter { + ($has_iter:ident $var:ident) => { + // `mut` may be unused if $var occurs multiple times in the list. + #[allow(unused_mut)] + let (mut $var, i) = $var.quote_into_iter(); + let $has_iter = $has_iter | i; + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_next_or_break { + ($var:ident) => { + let $var = match $var.next() { + Some(_x) => $crate::__private::RepInterp(_x), + None => break, + }; + }; +} + +// The obvious way to write this macro is as a tt muncher. This implementation +// does something more complex for two reasons. +// +// - With a tt muncher it's easy to hit Rust's built-in recursion_limit, which +// this implementation avoids because it isn't tail recursive. +// +// - Compile times for a tt muncher are quadratic relative to the length of +// the input. This implementation is linear, so it will be faster +// (potentially much faster) for big inputs. However, the constant factors +// of this implementation are higher than that of a tt muncher, so it is +// somewhat slower than a tt muncher if there are many invocations with +// short inputs. +// +// An invocation like this: +// +// quote_each_token!(_s a b c d e f g h i j); +// +// expands to this: +// +// quote_tokens_with_context!(_s +// (@ @ @ @ @ @ a b c d e f g h i j) +// (@ @ @ @ @ a b c d e f g h i j @) +// (@ @ @ @ a b c d e f g h i j @ @) +// (@ @ @ (a) (b) (c) (d) (e) (f) (g) (h) (i) (j) @ @ @) +// (@ @ a b c d e f g h i j @ @ @ @) +// (@ a b c d e f g h i j @ @ @ @ @) +// (a b c d e f g h i j @ @ @ @ @ @) +// ); +// +// which gets transposed and expanded to this: +// +// quote_token_with_context!(_s @ @ @ @ @ @ a); +// quote_token_with_context!(_s @ @ @ @ @ a b); +// quote_token_with_context!(_s @ @ @ @ a b c); +// quote_token_with_context!(_s @ @ @ (a) b c d); +// quote_token_with_context!(_s @ @ a (b) c d e); +// quote_token_with_context!(_s @ a b (c) d e f); +// quote_token_with_context!(_s a b c (d) e f g); +// quote_token_with_context!(_s b c d (e) f g h); +// quote_token_with_context!(_s c d e (f) g h i); +// quote_token_with_context!(_s d e f (g) h i j); +// quote_token_with_context!(_s e f g (h) i j @); +// quote_token_with_context!(_s f g h (i) j @ @); +// quote_token_with_context!(_s g h i (j) @ @ @); +// quote_token_with_context!(_s h i j @ @ @ @); +// quote_token_with_context!(_s i j @ @ @ @ @); +// quote_token_with_context!(_s j @ @ @ @ @ @); +// +// Without having used muncher-style recursion, we get one invocation of +// quote_token_with_context for each original tt, with three tts of context on +// either side. This is enough for the longest possible interpolation form (a +// repetition with separator, as in `# (#var) , *`) to be fully represented with +// the first or last tt in the middle. +// +// The middle tt (surrounded by parentheses) is the tt being processed. +// +// - When it is a `#`, quote_token_with_context can do an interpolation. The +// interpolation kind will depend on the three subsequent tts. +// +// - When it is within a later part of an interpolation, it can be ignored +// because the interpolation has already been done. +// +// - When it is not part of an interpolation it can be pushed as a single +// token into the output. +// +// - When the middle token is an unparenthesized `@`, that call is one of the +// first 3 or last 3 calls of quote_token_with_context and does not +// correspond to one of the original input tokens, so turns into nothing. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token { + ($tokens:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context!{$tokens + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token_spanned { + ($tokens:ident $span:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context_spanned!{$tokens $span + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context { + ($tokens:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context!{$tokens $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context_spanned { + ($tokens:ident $span:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context_spanned!{$tokens $span $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context { + // Unparenthesized `@` indicates this call does not correspond to one of the + // original input tokens. Ignore it. + ($tokens:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with no separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + // This is `while true` instead of `loop` because if there are no + // iterators used inside of this repetition then the body would not + // contain any `break`, so the compiler would emit unreachable code + // warnings on anything below the loop. We use has_iter to detect and + // fail to compile when there are no iterators, so here we just work + // around the unneeded extra warning. + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token!{$sep $tokens} + } + _i += 1; + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + // (A special case for `#(var)**`, where the first `*` is treated as the + // repetition symbol and the second `*` is treated as an ordinary token.) + ($tokens:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token!{* $tokens} + }; + // ... and one step later. + ($tokens:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A non-repetition interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + // An ordinary token, not part of any interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token!{$curr $tokens} + }; +} + +// See the explanation on quote_each_token, and on the individual rules of +// quote_token_with_context. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context_spanned { + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token_spanned!{$sep $tokens $span} + } + _i += 1; + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token_spanned!{* $tokens $span} + }; + ($tokens:ident $span:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + ($tokens:ident $span:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token_spanned!{$curr $tokens $span} + }; +} + +// These rules are ordered by approximate token frequency, at least for the +// first 10 or so, to improve compile times. Having `ident` first is by far the +// most important because it's typically 2-3x more common than the next most +// common token. +// +// Separately, we put the token being matched in the very front so that failing +// rules may fail to match as quickly as possible. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token { + ($ident:ident $tokens:ident) => { + $crate::__private::push_ident(&mut $tokens, stringify!($ident)); + }; + + (:: $tokens:ident) => { + $crate::__private::push_colon2(&mut $tokens); + }; + + (( $($inner:tt)* ) $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Parenthesis, + $crate::quote!($($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Bracket, + $crate::quote!($($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Brace, + $crate::quote!($($inner)*), + ); + }; + + (# $tokens:ident) => { + $crate::__private::push_pound(&mut $tokens); + }; + + (, $tokens:ident) => { + $crate::__private::push_comma(&mut $tokens); + }; + + (. $tokens:ident) => { + $crate::__private::push_dot(&mut $tokens); + }; + + (; $tokens:ident) => { + $crate::__private::push_semi(&mut $tokens); + }; + + (: $tokens:ident) => { + $crate::__private::push_colon(&mut $tokens); + }; + + (+ $tokens:ident) => { + $crate::__private::push_add(&mut $tokens); + }; + + (+= $tokens:ident) => { + $crate::__private::push_add_eq(&mut $tokens); + }; + + (& $tokens:ident) => { + $crate::__private::push_and(&mut $tokens); + }; + + (&& $tokens:ident) => { + $crate::__private::push_and_and(&mut $tokens); + }; + + (&= $tokens:ident) => { + $crate::__private::push_and_eq(&mut $tokens); + }; + + (@ $tokens:ident) => { + $crate::__private::push_at(&mut $tokens); + }; + + (! $tokens:ident) => { + $crate::__private::push_bang(&mut $tokens); + }; + + (^ $tokens:ident) => { + $crate::__private::push_caret(&mut $tokens); + }; + + (^= $tokens:ident) => { + $crate::__private::push_caret_eq(&mut $tokens); + }; + + (/ $tokens:ident) => { + $crate::__private::push_div(&mut $tokens); + }; + + (/= $tokens:ident) => { + $crate::__private::push_div_eq(&mut $tokens); + }; + + (.. $tokens:ident) => { + $crate::__private::push_dot2(&mut $tokens); + }; + + (... $tokens:ident) => { + $crate::__private::push_dot3(&mut $tokens); + }; + + (..= $tokens:ident) => { + $crate::__private::push_dot_dot_eq(&mut $tokens); + }; + + (= $tokens:ident) => { + $crate::__private::push_eq(&mut $tokens); + }; + + (== $tokens:ident) => { + $crate::__private::push_eq_eq(&mut $tokens); + }; + + (>= $tokens:ident) => { + $crate::__private::push_ge(&mut $tokens); + }; + + (> $tokens:ident) => { + $crate::__private::push_gt(&mut $tokens); + }; + + (<= $tokens:ident) => { + $crate::__private::push_le(&mut $tokens); + }; + + (< $tokens:ident) => { + $crate::__private::push_lt(&mut $tokens); + }; + + (*= $tokens:ident) => { + $crate::__private::push_mul_eq(&mut $tokens); + }; + + (!= $tokens:ident) => { + $crate::__private::push_ne(&mut $tokens); + }; + + (| $tokens:ident) => { + $crate::__private::push_or(&mut $tokens); + }; + + (|= $tokens:ident) => { + $crate::__private::push_or_eq(&mut $tokens); + }; + + (|| $tokens:ident) => { + $crate::__private::push_or_or(&mut $tokens); + }; + + (? $tokens:ident) => { + $crate::__private::push_question(&mut $tokens); + }; + + (-> $tokens:ident) => { + $crate::__private::push_rarrow(&mut $tokens); + }; + + (<- $tokens:ident) => { + $crate::__private::push_larrow(&mut $tokens); + }; + + (% $tokens:ident) => { + $crate::__private::push_rem(&mut $tokens); + }; + + (%= $tokens:ident) => { + $crate::__private::push_rem_eq(&mut $tokens); + }; + + (=> $tokens:ident) => { + $crate::__private::push_fat_arrow(&mut $tokens); + }; + + (<< $tokens:ident) => { + $crate::__private::push_shl(&mut $tokens); + }; + + (<<= $tokens:ident) => { + $crate::__private::push_shl_eq(&mut $tokens); + }; + + (>> $tokens:ident) => { + $crate::__private::push_shr(&mut $tokens); + }; + + (>>= $tokens:ident) => { + $crate::__private::push_shr_eq(&mut $tokens); + }; + + (* $tokens:ident) => { + $crate::__private::push_star(&mut $tokens); + }; + + (- $tokens:ident) => { + $crate::__private::push_sub(&mut $tokens); + }; + + (-= $tokens:ident) => { + $crate::__private::push_sub_eq(&mut $tokens); + }; + + ($lifetime:lifetime $tokens:ident) => { + $crate::__private::push_lifetime(&mut $tokens, stringify!($lifetime)); + }; + + (_ $tokens:ident) => { + $crate::__private::push_underscore(&mut $tokens); + }; + + ($other:tt $tokens:ident) => { + $crate::__private::parse(&mut $tokens, stringify!($other)); + }; +} + +// See the comment above `quote_token!` about the rule ordering. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_spanned { + ($ident:ident $tokens:ident $span:ident) => { + $crate::__private::push_ident_spanned(&mut $tokens, $span, stringify!($ident)); + }; + + (:: $tokens:ident $span:ident) => { + $crate::__private::push_colon2_spanned(&mut $tokens, $span); + }; + + (( $($inner:tt)* ) $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Parenthesis, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Bracket, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Brace, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + (# $tokens:ident $span:ident) => { + $crate::__private::push_pound_spanned(&mut $tokens, $span); + }; + + (, $tokens:ident $span:ident) => { + $crate::__private::push_comma_spanned(&mut $tokens, $span); + }; + + (. $tokens:ident $span:ident) => { + $crate::__private::push_dot_spanned(&mut $tokens, $span); + }; + + (; $tokens:ident $span:ident) => { + $crate::__private::push_semi_spanned(&mut $tokens, $span); + }; + + (: $tokens:ident $span:ident) => { + $crate::__private::push_colon_spanned(&mut $tokens, $span); + }; + + (+ $tokens:ident $span:ident) => { + $crate::__private::push_add_spanned(&mut $tokens, $span); + }; + + (+= $tokens:ident $span:ident) => { + $crate::__private::push_add_eq_spanned(&mut $tokens, $span); + }; + + (& $tokens:ident $span:ident) => { + $crate::__private::push_and_spanned(&mut $tokens, $span); + }; + + (&& $tokens:ident $span:ident) => { + $crate::__private::push_and_and_spanned(&mut $tokens, $span); + }; + + (&= $tokens:ident $span:ident) => { + $crate::__private::push_and_eq_spanned(&mut $tokens, $span); + }; + + (@ $tokens:ident $span:ident) => { + $crate::__private::push_at_spanned(&mut $tokens, $span); + }; + + (! $tokens:ident $span:ident) => { + $crate::__private::push_bang_spanned(&mut $tokens, $span); + }; + + (^ $tokens:ident $span:ident) => { + $crate::__private::push_caret_spanned(&mut $tokens, $span); + }; + + (^= $tokens:ident $span:ident) => { + $crate::__private::push_caret_eq_spanned(&mut $tokens, $span); + }; + + (/ $tokens:ident $span:ident) => { + $crate::__private::push_div_spanned(&mut $tokens, $span); + }; + + (/= $tokens:ident $span:ident) => { + $crate::__private::push_div_eq_spanned(&mut $tokens, $span); + }; + + (.. $tokens:ident $span:ident) => { + $crate::__private::push_dot2_spanned(&mut $tokens, $span); + }; + + (... $tokens:ident $span:ident) => { + $crate::__private::push_dot3_spanned(&mut $tokens, $span); + }; + + (..= $tokens:ident $span:ident) => { + $crate::__private::push_dot_dot_eq_spanned(&mut $tokens, $span); + }; + + (= $tokens:ident $span:ident) => { + $crate::__private::push_eq_spanned(&mut $tokens, $span); + }; + + (== $tokens:ident $span:ident) => { + $crate::__private::push_eq_eq_spanned(&mut $tokens, $span); + }; + + (>= $tokens:ident $span:ident) => { + $crate::__private::push_ge_spanned(&mut $tokens, $span); + }; + + (> $tokens:ident $span:ident) => { + $crate::__private::push_gt_spanned(&mut $tokens, $span); + }; + + (<= $tokens:ident $span:ident) => { + $crate::__private::push_le_spanned(&mut $tokens, $span); + }; + + (< $tokens:ident $span:ident) => { + $crate::__private::push_lt_spanned(&mut $tokens, $span); + }; + + (*= $tokens:ident $span:ident) => { + $crate::__private::push_mul_eq_spanned(&mut $tokens, $span); + }; + + (!= $tokens:ident $span:ident) => { + $crate::__private::push_ne_spanned(&mut $tokens, $span); + }; + + (| $tokens:ident $span:ident) => { + $crate::__private::push_or_spanned(&mut $tokens, $span); + }; + + (|= $tokens:ident $span:ident) => { + $crate::__private::push_or_eq_spanned(&mut $tokens, $span); + }; + + (|| $tokens:ident $span:ident) => { + $crate::__private::push_or_or_spanned(&mut $tokens, $span); + }; + + (? $tokens:ident $span:ident) => { + $crate::__private::push_question_spanned(&mut $tokens, $span); + }; + + (-> $tokens:ident $span:ident) => { + $crate::__private::push_rarrow_spanned(&mut $tokens, $span); + }; + + (<- $tokens:ident $span:ident) => { + $crate::__private::push_larrow_spanned(&mut $tokens, $span); + }; + + (% $tokens:ident $span:ident) => { + $crate::__private::push_rem_spanned(&mut $tokens, $span); + }; + + (%= $tokens:ident $span:ident) => { + $crate::__private::push_rem_eq_spanned(&mut $tokens, $span); + }; + + (=> $tokens:ident $span:ident) => { + $crate::__private::push_fat_arrow_spanned(&mut $tokens, $span); + }; + + (<< $tokens:ident $span:ident) => { + $crate::__private::push_shl_spanned(&mut $tokens, $span); + }; + + (<<= $tokens:ident $span:ident) => { + $crate::__private::push_shl_eq_spanned(&mut $tokens, $span); + }; + + (>> $tokens:ident $span:ident) => { + $crate::__private::push_shr_spanned(&mut $tokens, $span); + }; + + (>>= $tokens:ident $span:ident) => { + $crate::__private::push_shr_eq_spanned(&mut $tokens, $span); + }; + + (* $tokens:ident $span:ident) => { + $crate::__private::push_star_spanned(&mut $tokens, $span); + }; + + (- $tokens:ident $span:ident) => { + $crate::__private::push_sub_spanned(&mut $tokens, $span); + }; + + (-= $tokens:ident $span:ident) => { + $crate::__private::push_sub_eq_spanned(&mut $tokens, $span); + }; + + ($lifetime:lifetime $tokens:ident $span:ident) => { + $crate::__private::push_lifetime_spanned(&mut $tokens, $span, stringify!($lifetime)); + }; + + (_ $tokens:ident $span:ident) => { + $crate::__private::push_underscore_spanned(&mut $tokens, $span); + }; + + ($other:tt $tokens:ident $span:ident) => { + $crate::__private::parse_spanned(&mut $tokens, $span, stringify!($other)); + }; +} diff --git a/rust/quote/runtime.rs b/rust/quote/runtime.rs new file mode 100644 index 000000000..09a94f5dd --- /dev/null +++ b/rust/quote/runtime.rs @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use self::get_span::{GetSpan, GetSpanBase, GetSpanInner}; +use crate::{IdentFragment, ToTokens, TokenStreamExt}; +use core::fmt; +use core::iter; +use core::ops::BitOr; +use proc_macro2::{Group, Ident, Punct, Spacing, TokenTree}; + +#[doc(hidden)] +pub use alloc::format; +#[doc(hidden)] +pub use core::option::Option; + +#[doc(hidden)] +pub type Delimiter = proc_macro2::Delimiter; +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream = proc_macro2::TokenStream; + +#[doc(hidden)] +pub struct HasIterator; // True +#[doc(hidden)] +pub struct ThereIsNoIteratorInRepetition; // False + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = ThereIsNoIteratorInRepetition; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> ThereIsNoIteratorInRepetition { + ThereIsNoIteratorInRepetition + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> HasIterator { + HasIterator + } +} + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +/// Extension traits used by the implementation of `quote!`. These are defined +/// in separate traits, rather than as a single trait due to ambiguity issues. +/// +/// These traits expose a `quote_into_iter` method which should allow calling +/// whichever impl happens to be applicable. Calling that method repeatedly on +/// the returned value should be idempotent. +#[doc(hidden)] +pub mod ext { + use super::RepInterp; + use super::{HasIterator as HasIter, ThereIsNoIteratorInRepetition as DoesNotHaveIter}; + use crate::ToTokens; + use alloc::collections::btree_set::{self, BTreeSet}; + use core::slice; + + /// Extension trait providing the `quote_into_iter` method on iterators. + #[doc(hidden)] + pub trait RepIteratorExt: Iterator + Sized { + fn quote_into_iter(self) -> (Self, HasIter) { + (self, HasIter) + } + } + + impl RepIteratorExt for T {} + + /// Extension trait providing the `quote_into_iter` method for + /// non-iterable types. These types interpolate the same value in each + /// iteration of the repetition. + #[doc(hidden)] + pub trait RepToTokensExt { + /// Pretend to be an iterator for the purposes of `quote_into_iter`. + /// This allows repeated calls to `quote_into_iter` to continue + /// correctly returning DoesNotHaveIter. + fn next(&self) -> Option<&Self> { + Some(self) + } + + fn quote_into_iter(&self) -> (&Self, DoesNotHaveIter) { + (self, DoesNotHaveIter) + } + } + + impl RepToTokensExt for T {} + + /// Extension trait providing the `quote_into_iter` method for types that + /// can be referenced as an iterator. + #[doc(hidden)] + pub trait RepAsIteratorExt<'q> { + type Iter: Iterator; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter); + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &mut T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for [T] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q, const N: usize> RepAsIteratorExt<'q> for [T; N] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for Vec { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for BTreeSet { + type Iter = btree_set::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: RepAsIteratorExt<'q>> RepAsIteratorExt<'q> for RepInterp { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + self.0.quote_into_iter() + } + } +} + +// Helper type used within interpolations to allow for repeated binding names. +// Implements the relevant traits, and exports a dummy `next()` method. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct RepInterp(pub T); + +impl RepInterp { + // This method is intended to look like `Iterator::next`, and is called when + // a name is bound multiple times, as the previous binding will shadow the + // original `Iterator` object. This allows us to avoid advancing the + // iterator multiple times per iteration. + pub fn next(self) -> Option { + Some(self.0) + } +} + +impl Iterator for RepInterp { + type Item = T::Item; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +impl ToTokens for RepInterp { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens); + } +} + +#[doc(hidden)] +#[inline] +pub fn get_span(span: T) -> GetSpan { + GetSpan(GetSpanInner(GetSpanBase(span))) +} + +mod get_span { + use core::ops::Deref; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub struct GetSpan(pub(crate) GetSpanInner); + + pub struct GetSpanInner(pub(crate) GetSpanBase); + + pub struct GetSpanBase(pub(crate) T); + + impl GetSpan { + #[inline] + pub fn __into_span(self) -> Span { + ((self.0).0).0 + } + } + + impl GetSpanInner { + #[inline] + pub fn __into_span(&self) -> Span { + (self.0).0.join() + } + } + + impl GetSpanBase { + #[allow(clippy::unused_self)] + pub fn __into_span(&self) -> T { + unreachable!() + } + } + + impl Deref for GetSpan { + type Target = GetSpanInner; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl Deref for GetSpanInner { + type Target = GetSpanBase; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } +} + +#[doc(hidden)] +pub fn push_group(tokens: &mut TokenStream, delimiter: Delimiter, inner: TokenStream) { + tokens.append(Group::new(delimiter, inner)); +} + +#[doc(hidden)] +pub fn push_group_spanned( + tokens: &mut TokenStream, + span: Span, + delimiter: Delimiter, + inner: TokenStream, +) { + let mut g = Group::new(delimiter, inner); + g.set_span(span); + tokens.append(g); +} + +#[doc(hidden)] +pub fn parse(tokens: &mut TokenStream, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(iter::once(s)); +} + +#[doc(hidden)] +pub fn parse_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(s.into_iter().map(|t| respan_token_tree(t, span))); +} + +// Token tree with every span replaced by the given one. +fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree { + match &mut token { + TokenTree::Group(g) => { + let stream = g + .stream() + .into_iter() + .map(|token| respan_token_tree(token, span)) + .collect(); + *g = Group::new(g.delimiter(), stream); + g.set_span(span); + } + other => other.set_span(span), + } + token +} + +#[doc(hidden)] +pub fn push_ident(tokens: &mut TokenStream, s: &str) { + let span = Span::call_site(); + push_ident_spanned(tokens, span, s); +} + +#[doc(hidden)] +pub fn push_ident_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + tokens.append(ident_maybe_raw(s, span)); +} + +#[doc(hidden)] +pub fn push_lifetime(tokens: &mut TokenStream, lifetime: &str) { + tokens.extend([ + TokenTree::Punct(Punct::new('\'', Spacing::Joint)), + TokenTree::Ident(Ident::new(&lifetime[1..], Span::call_site())), + ]); +} + +#[doc(hidden)] +pub fn push_lifetime_spanned(tokens: &mut TokenStream, span: Span, lifetime: &str) { + tokens.extend([ + TokenTree::Punct({ + let mut apostrophe = Punct::new('\'', Spacing::Joint); + apostrophe.set_span(span); + apostrophe + }), + TokenTree::Ident(Ident::new(&lifetime[1..], span)), + ]); +} + +macro_rules! push_punct { + ($name:ident $spanned:ident $char1:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt $char3:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Joint)); + tokens.append(Punct::new($char3, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char3, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; +} + +push_punct!(push_add push_add_spanned '+'); +push_punct!(push_add_eq push_add_eq_spanned '+' '='); +push_punct!(push_and push_and_spanned '&'); +push_punct!(push_and_and push_and_and_spanned '&' '&'); +push_punct!(push_and_eq push_and_eq_spanned '&' '='); +push_punct!(push_at push_at_spanned '@'); +push_punct!(push_bang push_bang_spanned '!'); +push_punct!(push_caret push_caret_spanned '^'); +push_punct!(push_caret_eq push_caret_eq_spanned '^' '='); +push_punct!(push_colon push_colon_spanned ':'); +push_punct!(push_colon2 push_colon2_spanned ':' ':'); +push_punct!(push_comma push_comma_spanned ','); +push_punct!(push_div push_div_spanned '/'); +push_punct!(push_div_eq push_div_eq_spanned '/' '='); +push_punct!(push_dot push_dot_spanned '.'); +push_punct!(push_dot2 push_dot2_spanned '.' '.'); +push_punct!(push_dot3 push_dot3_spanned '.' '.' '.'); +push_punct!(push_dot_dot_eq push_dot_dot_eq_spanned '.' '.' '='); +push_punct!(push_eq push_eq_spanned '='); +push_punct!(push_eq_eq push_eq_eq_spanned '=' '='); +push_punct!(push_ge push_ge_spanned '>' '='); +push_punct!(push_gt push_gt_spanned '>'); +push_punct!(push_le push_le_spanned '<' '='); +push_punct!(push_lt push_lt_spanned '<'); +push_punct!(push_mul_eq push_mul_eq_spanned '*' '='); +push_punct!(push_ne push_ne_spanned '!' '='); +push_punct!(push_or push_or_spanned '|'); +push_punct!(push_or_eq push_or_eq_spanned '|' '='); +push_punct!(push_or_or push_or_or_spanned '|' '|'); +push_punct!(push_pound push_pound_spanned '#'); +push_punct!(push_question push_question_spanned '?'); +push_punct!(push_rarrow push_rarrow_spanned '-' '>'); +push_punct!(push_larrow push_larrow_spanned '<' '-'); +push_punct!(push_rem push_rem_spanned '%'); +push_punct!(push_rem_eq push_rem_eq_spanned '%' '='); +push_punct!(push_fat_arrow push_fat_arrow_spanned '=' '>'); +push_punct!(push_semi push_semi_spanned ';'); +push_punct!(push_shl push_shl_spanned '<' '<'); +push_punct!(push_shl_eq push_shl_eq_spanned '<' '<' '='); +push_punct!(push_shr push_shr_spanned '>' '>'); +push_punct!(push_shr_eq push_shr_eq_spanned '>' '>' '='); +push_punct!(push_star push_star_spanned '*'); +push_punct!(push_sub push_sub_spanned '-'); +push_punct!(push_sub_eq push_sub_eq_spanned '-' '='); + +#[doc(hidden)] +pub fn push_underscore(tokens: &mut TokenStream) { + push_underscore_spanned(tokens, Span::call_site()); +} + +#[doc(hidden)] +pub fn push_underscore_spanned(tokens: &mut TokenStream, span: Span) { + tokens.append(Ident::new("_", span)); +} + +// Helper method for constructing identifiers from the `format_ident!` macro, +// handling `r#` prefixes. +#[doc(hidden)] +pub fn mk_ident(id: &str, span: Option) -> Ident { + let span = span.unwrap_or_else(Span::call_site); + ident_maybe_raw(id, span) +} + +fn ident_maybe_raw(id: &str, span: Span) -> Ident { + if let Some(id) = id.strip_prefix("r#") { + Ident::new_raw(id, span) + } else { + Ident::new(id, span) + } +} + +// Adapts from `IdentFragment` to `fmt::Display` for use by the `format_ident!` +// macro, and exposes span information from these fragments. +// +// This struct also has forwarding implementations of the formatting traits +// `Octal`, `LowerHex`, `UpperHex`, and `Binary` to allow for their use within +// `format_ident!`. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct IdentFragmentAdapter(pub T); + +impl IdentFragmentAdapter { + pub fn span(&self) -> Option { + self.0.span() + } +} + +impl fmt::Display for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(&self.0, f) + } +} + +impl fmt::Octal for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Octal::fmt(&self.0, f) + } +} + +impl fmt::LowerHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} + +impl fmt::UpperHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl fmt::Binary for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Binary::fmt(&self.0, f) + } +} diff --git a/rust/quote/spanned.rs b/rust/quote/spanned.rs new file mode 100644 index 000000000..54ce9177f --- /dev/null +++ b/rust/quote/spanned.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::ToTokens; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Span, TokenStream}; + +// Not public API other than via the syn crate. Use syn::spanned::Spanned. +pub trait Spanned: private::Sealed { + fn __span(&self) -> Span; +} + +impl Spanned for Span { + fn __span(&self) -> Span { + *self + } +} + +impl Spanned for DelimSpan { + fn __span(&self) -> Span { + self.join() + } +} + +impl Spanned for T { + fn __span(&self) -> Span { + join_spans(self.into_token_stream()) + } +} + +fn join_spans(tokens: TokenStream) -> Span { + let mut iter = tokens.into_iter().map(|tt| tt.span()); + + let first = match iter.next() { + Some(span) => span, + None => return Span::call_site(), + }; + + iter.fold(None, |_prev, next| Some(next)) + .and_then(|last| first.join(last)) + .unwrap_or(first) +} + +mod private { + use crate::ToTokens; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub trait Sealed {} + impl Sealed for Span {} + impl Sealed for DelimSpan {} + impl Sealed for T {} +} diff --git a/rust/quote/to_tokens.rs b/rust/quote/to_tokens.rs new file mode 100644 index 000000000..1af1089e1 --- /dev/null +++ b/rust/quote/to_tokens.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use super::TokenStreamExt; +use alloc::borrow::Cow; +use alloc::rc::Rc; +use core::iter; +use proc_macro2::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree}; +use std::ffi::{CStr, CString}; + +/// Types that can be interpolated inside a `quote!` invocation. +pub trait ToTokens { + /// Write `self` to the given `TokenStream`. + /// + /// The token append methods provided by the [`TokenStreamExt`] extension + /// trait may be useful for implementing `ToTokens`. + /// + /// # Example + /// + /// Example implementation for a struct representing Rust paths like + /// `std::cmp::PartialEq`: + /// + /// ``` + /// use proc_macro2::{TokenTree, Spacing, Span, Punct, TokenStream}; + /// use quote::{TokenStreamExt, ToTokens}; + /// + /// pub struct Path { + /// pub global: bool, + /// pub segments: Vec, + /// } + /// + /// impl ToTokens for Path { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// for (i, segment) in self.segments.iter().enumerate() { + /// if i > 0 || self.global { + /// // Double colon `::` + /// tokens.append(Punct::new(':', Spacing::Joint)); + /// tokens.append(Punct::new(':', Spacing::Alone)); + /// } + /// segment.to_tokens(tokens); + /// } + /// } + /// } + /// # + /// # pub struct PathSegment; + /// # + /// # impl ToTokens for PathSegment { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # unimplemented!() + /// # } + /// # } + /// ``` + fn to_tokens(&self, tokens: &mut TokenStream); + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn to_token_stream(&self) -> TokenStream { + let mut tokens = TokenStream::new(); + self.to_tokens(&mut tokens); + tokens + } + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn into_token_stream(self) -> TokenStream + where + Self: Sized, + { + self.to_token_stream() + } +} + +impl ToTokens for &T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for &mut T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Box { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Rc { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Option { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Some(t) = self { + t.to_tokens(tokens); + } + } +} + +impl ToTokens for str { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::string(self)); + } +} + +impl ToTokens for String { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.as_str().to_tokens(tokens); + } +} + +impl ToTokens for i8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i8_suffixed(*self)); + } +} + +impl ToTokens for i16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i16_suffixed(*self)); + } +} + +impl ToTokens for i32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i32_suffixed(*self)); + } +} + +impl ToTokens for i64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i64_suffixed(*self)); + } +} + +impl ToTokens for i128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i128_suffixed(*self)); + } +} + +impl ToTokens for isize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::isize_suffixed(*self)); + } +} + +impl ToTokens for u8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u8_suffixed(*self)); + } +} + +impl ToTokens for u16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u16_suffixed(*self)); + } +} + +impl ToTokens for u32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u32_suffixed(*self)); + } +} + +impl ToTokens for u64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u64_suffixed(*self)); + } +} + +impl ToTokens for u128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u128_suffixed(*self)); + } +} + +impl ToTokens for usize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::usize_suffixed(*self)); + } +} + +impl ToTokens for f32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f32_suffixed(*self)); + } +} + +impl ToTokens for f64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f64_suffixed(*self)); + } +} + +impl ToTokens for char { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::character(*self)); + } +} + +impl ToTokens for bool { + fn to_tokens(&self, tokens: &mut TokenStream) { + let word = if *self { "true" } else { "false" }; + tokens.append(Ident::new(word, Span::call_site())); + } +} + +impl ToTokens for CStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for CString { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for Group { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Ident { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Punct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Literal { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenTree { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenStream { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(iter::once(self.clone())); + } + + fn into_token_stream(self) -> TokenStream { + self + } +} diff --git a/rust/syn/README.md b/rust/syn/README.md new file mode 100644 index 000000000..d3e3981da --- /dev/null +++ b/rust/syn/README.md @@ -0,0 +1,13 @@ +# `syn` + +These source files come from the Rust `syn` crate, version 2.0.106 +(released 2025-08-16), hosted in the +repository, licensed under "Apache-2.0 OR MIT" and only modified to add +the SPDX license identifiers and to remove the `unicode-ident` +dependency. + +For copyright details, please see: + + https://github.com/dtolnay/syn/blob/2.0.106/README.md#license + https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-APACHE + https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-MIT diff --git a/rust/syn/attr.rs b/rust/syn/attr.rs new file mode 100644 index 000000000..020b31f9d --- /dev/null +++ b/rust/syn/attr.rs @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "parsing")] +use crate::error::Error; +#[cfg(feature = "parsing")] +use crate::error::Result; +use crate::expr::Expr; +use crate::mac::MacroDelimiter; +#[cfg(feature = "parsing")] +use crate::meta::{self, ParseNestedMeta}; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream, Parser}; +use crate::path::Path; +use crate::token; +use proc_macro2::TokenStream; +#[cfg(feature = "printing")] +use std::iter; +#[cfg(feature = "printing")] +use std::slice; + +ast_struct! { + /// An attribute, like `#[repr(transparent)]`. + /// + ///
+ /// + /// # Syntax + /// + /// Rust has six types of attributes. + /// + /// - Outer attributes like `#[repr(transparent)]`. These appear outside or + /// in front of the item they describe. + /// + /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside + /// of the item they describe, usually a module. + /// + /// - Outer one-line doc comments like `/// Example`. + /// + /// - Inner one-line doc comments like `//! Please file an issue`. + /// + /// - Outer documentation blocks `/** Example */`. + /// + /// - Inner documentation blocks `/*! Please file an issue */`. + /// + /// The `style` field of type `AttrStyle` distinguishes whether an attribute + /// is outer or inner. + /// + /// Every attribute has a `path` that indicates the intended interpretation + /// of the rest of the attribute's contents. The path and the optional + /// additional contents are represented together in the `meta` field of the + /// attribute in three possible varieties: + /// + /// - Meta::Path — attributes whose information content conveys just a + /// path, for example the `#[test]` attribute. + /// + /// - Meta::List — attributes that carry arbitrary tokens after the + /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For + /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. + /// + /// - Meta::NameValue — attributes with an `=` sign after the path, + /// followed by a Rust expression. For example `#[path = + /// "sys/windows.rs"]`. + /// + /// All doc comments are represented in the NameValue style with a path of + /// "doc", as this is how they are processed by the compiler and by + /// `macro_rules!` macros. + /// + /// ```text + /// #[derive(Copy, Clone)] + /// ~~~~~~Path + /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// + /// #[path = "sys/windows.rs"] + /// ~~~~Path + /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue + /// + /// #[test] + /// ^^^^Meta::Path + /// ``` + /// + ///
+ /// + /// # Parsing from tokens to Attribute + /// + /// This type does not implement the [`Parse`] trait and thus cannot be + /// parsed directly by [`ParseStream::parse`]. Instead use + /// [`ParseStream::call`] with one of the two parser functions + /// [`Attribute::parse_outer`] or [`Attribute::parse_inner`] depending on + /// which you intend to parse. + /// + /// [`Parse`]: crate::parse::Parse + /// [`ParseStream::parse`]: crate::parse::ParseBuffer::parse + /// [`ParseStream::call`]: crate::parse::ParseBuffer::call + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + /// + ///


+ /// + /// # Parsing from Attribute to structured arguments + /// + /// The grammar of attributes in Rust is very flexible, which makes the + /// syntax tree not that useful on its own. In particular, arguments of the + /// `Meta::List` variety of attribute are held in an arbitrary `tokens: + /// TokenStream`. Macros are expected to check the `path` of the attribute, + /// decide whether they recognize it, and then parse the remaining tokens + /// according to whatever grammar they wish to require for that kind of + /// attribute. Use [`parse_args()`] to parse those tokens into the expected + /// data structure. + /// + /// [`parse_args()`]: Attribute::parse_args + /// + ///


+ /// + /// # Doc comments + /// + /// The compiler transforms doc comments, such as `/// comment` and `/*! + /// comment */`, into attributes before macros are expanded. Each comment is + /// expanded into an attribute of the form `#[doc = r"comment"]`. + /// + /// As an example, the following `mod` items are expanded identically: + /// + /// ``` + /// # use syn::{ItemMod, parse_quote}; + /// let doc: ItemMod = parse_quote! { + /// /// Single line doc comments + /// /// We write so many! + /// /** + /// * Multi-line comments... + /// * May span many lines + /// */ + /// mod example { + /// //! Of course, they can be inner too + /// /*! And fit in a single line */ + /// } + /// }; + /// let attr: ItemMod = parse_quote! { + /// #[doc = r" Single line doc comments"] + /// #[doc = r" We write so many!"] + /// #[doc = r" + /// * Multi-line comments... + /// * May span many lines + /// "] + /// mod example { + /// #![doc = r" Of course, they can be inner too"] + /// #![doc = r" And fit in a single line "] + /// } + /// }; + /// assert_eq!(doc, attr); + /// ``` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Attribute { + pub pound_token: Token![#], + pub style: AttrStyle, + pub bracket_token: token::Bracket, + pub meta: Meta, + } +} + +impl Attribute { + /// Returns the path that identifies the interpretation of this attribute. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + self.meta.path() + } + + /// Parse the arguments to the attribute as a syntax tree. + /// + /// This is similar to pulling out the `TokenStream` from `Meta::List` and + /// doing `syn::parse2::(meta_list.tokens)`, except that using + /// `parse_args` the error message has a more useful span when `tokens` is + /// empty. + /// + /// The surrounding delimiters are *not* included in the input to the + /// parser. + /// + /// ```text + /// #[my_attr(value < 5)] + /// ^^^^^^^^^ what gets parsed + /// ``` + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, Expr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[precondition(value < 5)] + /// }; + /// + /// if attr.path().is_ident("precondition") { + /// let precondition: Expr = attr.parse_args()?; + /// // ... + /// } + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// Parse the arguments to the attribute using the given parser. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] + /// }; + /// + /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; + /// + /// // Attribute does not have a Parse impl, so we couldn't directly do: + /// // let bwom: Attribute = attr.parse_args()?; + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + match &self.meta { + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new( + meta.eq_token.span, + format_args!( + "expected parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(&meta.path), + ), + )), + Meta::List(meta) => meta.parse_args_with(parser), + } + } + + /// Parse the arguments to the attribute, expecting it to follow the + /// conventional structure used by most of Rust's built-in attributes. + /// + /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference + /// explains the convention in more detail. Not all attributes follow this + /// convention, so [`parse_args()`][Self::parse_args] is available if you + /// need to parse arbitrarily goofy attribute syntax. + /// + /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax + /// + /// # Example + /// + /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute + /// syntax. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; + /// + /// let input: ItemStruct = parse_quote! { + /// #[repr(C, align(4))] + /// pub struct MyStruct(u16, u32); + /// }; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// // #[repr(C)] + /// if meta.path.is_ident("C") { + /// repr_c = true; + /// return Ok(()); + /// } + /// + /// // #[repr(transparent)] + /// if meta.path.is_ident("transparent") { + /// repr_transparent = true; + /// return Ok(()); + /// } + /// + /// // #[repr(align(N))] + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// return Ok(()); + /// } + /// + /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 + /// if meta.path.is_ident("packed") { + /// if meta.input.peek(token::Paren) { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_packed = Some(n); + /// } else { + /// repr_packed = Some(1); + /// } + /// return Ok(()); + /// } + /// + /// Err(meta.error("unrecognized repr")) + /// })?; + /// } + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Alternatives + /// + /// In some cases, for attributes which have nested layers of structured + /// content, the following less flexible approach might be more convenient: + /// + /// ``` + /// # use syn::{parse_quote, ItemStruct}; + /// # + /// # let input: ItemStruct = parse_quote! { + /// # #[repr(C, align(4))] + /// # pub struct MyStruct(u16, u32); + /// # }; + /// # + /// use syn::punctuated::Punctuated; + /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// let nested = attr.parse_args_with(Punctuated::::parse_terminated)?; + /// for meta in nested { + /// match meta { + /// // #[repr(C)] + /// Meta::Path(path) if path.is_ident("C") => { + /// repr_c = true; + /// } + /// + /// // #[repr(align(N))] + /// Meta::List(meta) if meta.path.is_ident("align") => { + /// let lit: LitInt = meta.parse_args()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// } + /// + /// /* ... */ + /// + /// _ => { + /// return Err(Error::new_spanned(meta, "unrecognized repr")); + /// } + /// } + /// } + /// } + /// } + /// # Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } + + /// Parses zero or more outer attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_outer(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) { + attrs.push(input.call(parsing::single_parse_outer)?); + } + Ok(attrs) + } + + /// Parses zero or more inner attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_inner(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + parsing::parse_inner(input, &mut attrs)?; + Ok(attrs) + } +} + +ast_enum! { + /// Distinguishes between attributes that decorate an item and attributes + /// that are contained within an item. + /// + /// # Outer attributes + /// + /// - `#[repr(transparent)]` + /// - `/// # Example` + /// - `/** Please file an issue */` + /// + /// # Inner attributes + /// + /// - `#![feature(proc_macro)]` + /// - `//! # Example` + /// - `/*! Please file an issue */` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum AttrStyle { + Outer, + Inner(Token![!]), + } +} + +ast_enum! { + /// Content of a compile-time structured attribute. + /// + /// ## Path + /// + /// A meta path is like the `test` in `#[test]`. + /// + /// ## List + /// + /// A meta list is like the `derive(Copy)` in `#[derive(Copy)]`. + /// + /// ## NameValue + /// + /// A name-value meta is like the `path = "..."` in `#[path = + /// "sys/windows.rs"]`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Meta { + Path(Path), + + /// A structured list within an attribute, like `derive(Copy, Clone)`. + List(MetaList), + + /// A name-value pair within an attribute, like `feature = "nightly"`. + NameValue(MetaNameValue), + } +} + +ast_struct! { + /// A structured list within an attribute, like `derive(Copy, Clone)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaList { + pub path: Path, + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, + } +} + +ast_struct! { + /// A name-value pair within an attribute, like `feature = "nightly"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaNameValue { + pub path: Path, + pub eq_token: Token![=], + pub value: Expr, + } +} + +impl Meta { + /// Returns the path that begins this structured meta item. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + match self { + Meta::Path(path) => path, + Meta::List(meta) => &meta.path, + Meta::NameValue(meta) => &meta.path, + } + } + + /// Error if this is a `Meta::List` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_path_only(&self) -> Result<&Path> { + let error_span = match self { + Meta::Path(path) => return Ok(path), + Meta::List(meta) => meta.delimiter.span().open(), + Meta::NameValue(meta) => meta.eq_token.span, + }; + Err(Error::new(error_span, "unexpected token in attribute")) + } + + /// Error if this is a `Meta::Path` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_list(&self) -> Result<&MetaList> { + match self { + Meta::List(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: `{}(...)`", + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), + } + } + + /// Error if this is a `Meta::Path` or `Meta::List`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_name_value(&self) -> Result<&MetaNameValue> { + match self { + Meta::NameValue(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected a value for this attribute: `{} = ...`", + parsing::DisplayPath(path), + ), + )), + Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), + } + } +} + +impl MetaList { + /// See [`Attribute::parse_args`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// See [`Attribute::parse_args_with`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } + + /// See [`Attribute::parse_nested_meta`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } +} + +#[cfg(feature = "printing")] +pub(crate) trait FilterAttrs<'a> { + type Ret: Iterator; + + fn outer(self) -> Self::Ret; + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret; +} + +#[cfg(feature = "printing")] +impl<'a> FilterAttrs<'a> for &'a [Attribute] { + type Ret = iter::Filter, fn(&&Attribute) -> bool>; + + fn outer(self) -> Self::Ret { + fn is_outer(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Outer => true, + AttrStyle::Inner(_) => false, + } + } + self.iter().filter(is_outer) + } + + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret { + fn is_inner(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Inner(_) => true, + AttrStyle::Outer => false, + } + } + self.iter().filter(is_inner) + } +} + +impl From for Meta { + fn from(meta: Path) -> Meta { + Meta::Path(meta) + } +} + +impl From for Meta { + fn from(meta: MetaList) -> Meta { + Meta::List(meta) + } +} + +impl From for Meta { + fn from(meta: MetaNameValue) -> Meta { + Meta::NameValue(meta) + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::error::Result; + use crate::expr::{Expr, ExprLit}; + use crate::lit::Lit; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + use crate::{mac, token}; + use proc_macro2::Ident; + use std::fmt::{self, Display}; + + pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { + while input.peek(Token![#]) && input.peek2(Token![!]) { + attrs.push(input.call(single_parse_inner)?); + } + Ok(()) + } + + pub(crate) fn single_parse_inner(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Inner(input.parse()?), + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + pub(crate) fn single_parse_outer(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Outer, + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Meta { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaList { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_list_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaNameValue { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_name_value_after_path(path, input) + } + } + + // Unlike meta::parse_meta_path which accepts arbitrary keywords in the path, + // only the `unsafe` keyword is accepted as an attribute's outermost path. + fn parse_outermost_meta_path(input: ParseStream) -> Result { + if input.peek(Token![unsafe]) { + let unsafe_token: Token![unsafe] = input.parse()?; + Ok(Path::from(Ident::new("unsafe", unsafe_token.span))) + } else { + Path::parse_mod_style(input) + } + } + + pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { + parse_meta_list_after_path(path, input).map(Meta::List) + } else if input.peek(Token![=]) { + parse_meta_name_value_after_path(path, input).map(Meta::NameValue) + } else { + Ok(Meta::Path(path)) + } + } + + fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { + let (delimiter, tokens) = mac::parse_delimiter(input)?; + Ok(MetaList { + path, + delimiter, + tokens, + }) + } + + fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { + let eq_token: Token![=] = input.parse()?; + let ahead = input.fork(); + let lit: Option = ahead.parse()?; + let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { + input.advance_to(&ahead); + Expr::Lit(ExprLit { + attrs: Vec::new(), + lit, + }) + } else if input.peek(Token![#]) && input.peek2(token::Bracket) { + return Err(input.error("unexpected attribute inside of attribute")); + } else { + input.parse()? + }; + Ok(MetaNameValue { + path, + eq_token, + value, + }) + } + + pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); + + impl<'a> Display for DisplayAttrStyle<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(match self.0 { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }) + } + } + + pub(super) struct DisplayPath<'a>(pub &'a Path); + + impl<'a> Display for DisplayPath<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + for (i, segment) in self.0.segments.iter().enumerate() { + if i > 0 || self.0.leading_colon.is_some() { + formatter.write_str("::")?; + } + write!(formatter, "{}", segment.ident)?; + } + Ok(()) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::path; + use crate::path::printing::PathStyle; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Attribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pound_token.to_tokens(tokens); + if let AttrStyle::Inner(b) = &self.style { + b.to_tokens(tokens); + } + self.bracket_token.surround(tokens, |tokens| { + self.meta.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Meta { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Meta::Path(path) => path::printing::print_path(tokens, path, PathStyle::Mod), + Meta::List(meta_list) => meta_list.to_tokens(tokens), + Meta::NameValue(meta_name_value) => meta_name_value.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaList { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.delimiter.surround(tokens, self.tokens.clone()); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaNameValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.eq_token.to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} diff --git a/rust/syn/bigint.rs b/rust/syn/bigint.rs new file mode 100644 index 000000000..d706f9f51 --- /dev/null +++ b/rust/syn/bigint.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::ops::{AddAssign, MulAssign}; + +// For implementing base10_digits() accessor on LitInt. +pub(crate) struct BigInt { + digits: Vec, +} + +impl BigInt { + pub(crate) fn new() -> Self { + BigInt { digits: Vec::new() } + } + + pub(crate) fn to_string(&self) -> String { + let mut repr = String::with_capacity(self.digits.len()); + + let mut has_nonzero = false; + for digit in self.digits.iter().rev() { + has_nonzero |= *digit != 0; + if has_nonzero { + repr.push((*digit + b'0') as char); + } + } + + if repr.is_empty() { + repr.push('0'); + } + + repr + } + + fn reserve_two_digits(&mut self) { + let len = self.digits.len(); + let desired = + len + !self.digits.ends_with(&[0, 0]) as usize + !self.digits.ends_with(&[0]) as usize; + self.digits.resize(desired, 0); + } +} + +impl AddAssign for BigInt { + // Assumes increment <16. + fn add_assign(&mut self, mut increment: u8) { + self.reserve_two_digits(); + + let mut i = 0; + while increment > 0 { + let sum = self.digits[i] + increment; + self.digits[i] = sum % 10; + increment = sum / 10; + i += 1; + } + } +} + +impl MulAssign for BigInt { + // Assumes base <=16. + fn mul_assign(&mut self, base: u8) { + self.reserve_two_digits(); + + let mut carry = 0; + for digit in &mut self.digits { + let prod = *digit * base + carry; + *digit = prod % 10; + carry = prod / 10; + } + } +} diff --git a/rust/syn/buffer.rs b/rust/syn/buffer.rs new file mode 100644 index 000000000..406f70db3 --- /dev/null +++ b/rust/syn/buffer.rs @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A stably addressed token buffer supporting efficient traversal based on a +//! cheaply copyable cursor. + +// This module is heavily commented as it contains most of the unsafe code in +// Syn, and caution should be used when editing it. The public-facing interface +// is 100% safe but the implementation is fragile internally. + +use crate::Lifetime; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::cmp::Ordering; +use std::marker::PhantomData; +use std::ptr; + +/// Internal type which is used instead of `TokenTree` to represent a token tree +/// within a `TokenBuffer`. +enum Entry { + // Mimicking types from proc-macro. + // Group entries contain the offset to the matching End entry. + Group(Group, usize), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain the offset (negative) to the start of the buffer, and + // offset (negative) to the matching Group entry. + End(isize, isize), +} + +/// A buffer that can be efficiently traversed multiple times, unlike +/// `TokenStream` which requires a deep copy in order to traverse more than +/// once. +pub struct TokenBuffer { + // NOTE: Do not implement clone on this - while the current design could be + // cloned, other designs which could be desirable may not be cloneable. + entries: Box<[Entry]>, +} + +impl TokenBuffer { + fn recursive_new(entries: &mut Vec, stream: TokenStream) { + for tt in stream { + match tt { + TokenTree::Ident(ident) => entries.push(Entry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(Entry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(Entry::Literal(literal)), + TokenTree::Group(group) => { + let group_start_index = entries.len(); + entries.push(Entry::End(0, 0)); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + let group_offset = group_end_index - group_start_index; + entries.push(Entry::End( + -(group_end_index as isize), + -(group_offset as isize), + )); + entries[group_start_index] = Entry::Group(group, group_offset); + } + } + } + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro::TokenStream`. + #[cfg(feature = "proc-macro")] + #[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] + pub fn new(stream: proc_macro::TokenStream) -> Self { + Self::new2(stream.into()) + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro2::TokenStream`. + pub fn new2(stream: TokenStream) -> Self { + let mut entries = Vec::new(); + Self::recursive_new(&mut entries, stream); + entries.push(Entry::End(-(entries.len() as isize), 0)); + Self { + entries: entries.into_boxed_slice(), + } + } + + /// Creates a cursor referencing the first token in the buffer and able to + /// traverse until the end of the buffer. + pub fn begin(&self) -> Cursor { + let ptr = self.entries.as_ptr(); + unsafe { Cursor::create(ptr, ptr.add(self.entries.len() - 1)) } + } +} + +/// A cheaply copyable cursor into a `TokenBuffer`. +/// +/// This cursor holds a shared reference into the immutable data which is used +/// internally to represent a `TokenStream`, and can be efficiently manipulated +/// and copied around. +/// +/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` +/// object and get a cursor to its first token with `begin()`. +pub struct Cursor<'a> { + // The current entry which the `Cursor` is pointing at. + ptr: *const Entry, + // This is the only `Entry::End` object which this cursor is allowed to + // point at. All other `End` objects are skipped over in `Cursor::create`. + scope: *const Entry, + // Cursor is covariant in 'a. This field ensures that our pointers are still + // valid. + marker: PhantomData<&'a Entry>, +} + +impl<'a> Cursor<'a> { + /// Creates a cursor referencing a static empty TokenStream. + pub fn empty() -> Self { + // It's safe in this situation for us to put an `Entry` object in global + // storage, despite it not actually being safe to send across threads + // (`Ident` is a reference into a thread-local table). This is because + // this entry never includes a `Ident` object. + // + // This wrapper struct allows us to break the rules and put a `Sync` + // object in global storage. + struct UnsafeSyncEntry(Entry); + unsafe impl Sync for UnsafeSyncEntry {} + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0, 0)); + + Cursor { + ptr: &EMPTY_ENTRY.0, + scope: &EMPTY_ENTRY.0, + marker: PhantomData, + } + } + + /// This create method intelligently exits non-explicitly-entered + /// `None`-delimited scopes when the cursor reaches the end of them, + /// allowing for them to be treated transparently. + unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { + // NOTE: If we're looking at a `End`, we want to advance the cursor + // past it, unless `ptr == scope`, which means that we're at the edge of + // our cursor's scope. We should only have `ptr != scope` at the exit + // from None-delimited groups entered with `ignore_none`. + while let Entry::End(..) = unsafe { &*ptr } { + if ptr::eq(ptr, scope) { + break; + } + ptr = unsafe { ptr.add(1) }; + } + + Cursor { + ptr, + scope, + marker: PhantomData, + } + } + + /// Get the current entry. + fn entry(self) -> &'a Entry { + unsafe { &*self.ptr } + } + + /// Bump the cursor to point at the next token after the current one. This + /// is undefined behavior if the cursor is currently looking at an + /// `Entry::End`. + /// + /// If the cursor is looking at an `Entry::Group`, the bumped cursor will + /// point at the first token in the group (with the same scope end). + unsafe fn bump_ignore_group(self) -> Cursor<'a> { + unsafe { Cursor::create(self.ptr.offset(1), self.scope) } + } + + /// While the cursor is looking at a `None`-delimited group, move it to look + /// at the first token inside instead. If the group is empty, this will move + /// the cursor past the `None`-delimited group. + /// + /// WARNING: This mutates its argument. + fn ignore_none(&mut self) { + while let Entry::Group(group, _) = self.entry() { + if group.delimiter() == Delimiter::None { + unsafe { *self = self.bump_ignore_group() }; + } else { + break; + } + } + } + + /// Checks whether the cursor is currently pointing at the end of its valid + /// scope. + pub fn eof(self) -> bool { + // We're at eof if we're at the end of our scope. + ptr::eq(self.ptr, self.scope) + } + + /// If the cursor is pointing at a `Ident`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Punct`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() != '\'' => { + Some((punct.clone(), unsafe { self.bump_ignore_group() })) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Literal`, return it along with a cursor + /// pointing at the next `TokenTree`. + pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Lifetime`, returns it along with a + /// cursor pointing at the next `TokenTree`. + pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + let next = unsafe { self.bump_ignore_group() }; + let (ident, rest) = next.ident()?; + let lifetime = Lifetime { + apostrophe: punct.span(), + ident, + }; + Some((lifetime, rest)) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Group` with the given delimiter, returns + /// a cursor into that group and one pointing to the next `TokenTree`. + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { + // If we're not trying to enter a none-delimited group, we want to + // ignore them. We have to make sure to _not_ ignore them when we want + // to enter them, of course. For obvious reasons. + if delim != Delimiter::None { + self.ignore_none(); + } + + if let Entry::Group(group, end_offset) = self.entry() { + if group.delimiter() == delim { + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, span, after_group)); + } + } + + None + } + + /// If the cursor is pointing at a `Group`, returns a cursor into the group + /// and one pointing to the next `TokenTree`. + pub fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let delimiter = group.delimiter(); + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, delimiter, span, after_group)); + } + + None + } + + pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((group.clone(), after_group)); + } + + None + } + + /// Copies all remaining tokens visible from this cursor into a + /// `TokenStream`. + pub fn token_stream(self) -> TokenStream { + let mut tts = Vec::new(); + let mut cursor = self; + while let Some((tt, rest)) = cursor.token_tree() { + tts.push(tt); + cursor = rest; + } + tts.into_iter().collect() + } + + /// If the cursor is pointing at a `TokenTree`, returns it along with a + /// cursor pointing at the next `TokenTree`. + /// + /// Returns `None` if the cursor has reached the end of its stream. + /// + /// This method does not treat `None`-delimited groups as transparent, and + /// will return a `Group(None, ..)` if the cursor is looking at one. + pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { + let (tree, len) = match self.entry() { + Entry::Group(group, end_offset) => (group.clone().into(), *end_offset), + Entry::Literal(literal) => (literal.clone().into(), 1), + Entry::Ident(ident) => (ident.clone().into(), 1), + Entry::Punct(punct) => (punct.clone().into(), 1), + Entry::End(..) => return None, + }; + + let rest = unsafe { Cursor::create(self.ptr.add(len), self.scope) }; + Some((tree, rest)) + } + + /// Returns the `Span` of the current token, or `Span::call_site()` if this + /// cursor points to eof. + pub fn span(mut self) -> Span { + match self.entry() { + Entry::Group(group, _) => group.span(), + Entry::Literal(literal) => literal.span(), + Entry::Ident(ident) => ident.span(), + Entry::Punct(punct) => punct.span(), + Entry::End(_, offset) => { + self.ptr = unsafe { self.ptr.offset(*offset) }; + if let Entry::Group(group, _) = self.entry() { + group.span_close() + } else { + Span::call_site() + } + } + } + } + + /// Returns the `Span` of the token immediately prior to the position of + /// this cursor, or of the current token if there is no previous one. + #[cfg(any(feature = "full", feature = "derive"))] + pub(crate) fn prev_span(mut self) -> Span { + if start_of_buffer(self) < self.ptr { + self.ptr = unsafe { self.ptr.offset(-1) }; + } + self.span() + } + + /// Skip over the next token that is not a None-delimited group, without + /// cloning it. Returns `None` if this cursor points to eof. + /// + /// This method treats `'lifetimes` as a single token. + pub(crate) fn skip(mut self) -> Option> { + self.ignore_none(); + + let len = match self.entry() { + Entry::End(..) => return None, + + // Treat lifetimes as a single tt for the purposes of 'skip'. + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + match unsafe { &*self.ptr.add(1) } { + Entry::Ident(_) => 2, + _ => 1, + } + } + + Entry::Group(_, end_offset) => *end_offset, + _ => 1, + }; + + Some(unsafe { Cursor::create(self.ptr.add(len), self.scope) }) + } + + pub(crate) fn scope_delimiter(self) -> Delimiter { + match unsafe { &*self.scope } { + Entry::End(_, offset) => match unsafe { &*self.scope.offset(*offset) } { + Entry::Group(group, _) => group.delimiter(), + _ => Delimiter::None, + }, + _ => unreachable!(), + } + } +} + +impl<'a> Copy for Cursor<'a> {} + +impl<'a> Clone for Cursor<'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a> Eq for Cursor<'a> {} + +impl<'a> PartialEq for Cursor<'a> { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.ptr, other.ptr) + } +} + +impl<'a> PartialOrd for Cursor<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + if same_buffer(*self, *other) { + Some(cmp_assuming_same_buffer(*self, *other)) + } else { + None + } + } +} + +pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { + ptr::eq(a.scope, b.scope) +} + +pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { + ptr::eq(start_of_buffer(a), start_of_buffer(b)) +} + +fn start_of_buffer(cursor: Cursor) -> *const Entry { + unsafe { + match &*cursor.scope { + Entry::End(offset, _) => cursor.scope.offset(*offset), + _ => unreachable!(), + } + } +} + +pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { + a.ptr.cmp(&b.ptr) +} + +pub(crate) fn open_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_open(), + _ => cursor.span(), + } +} diff --git a/rust/syn/classify.rs b/rust/syn/classify.rs new file mode 100644 index 000000000..92a975bfe --- /dev/null +++ b/rust/syn/classify.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "full")] +use crate::expr::Expr; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::generics::TypeParamBound; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::path::{Path, PathArguments}; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::punctuated::Punctuated; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::ty::{ReturnType, Type}; +#[cfg(feature = "full")] +use proc_macro2::{Delimiter, TokenStream, TokenTree}; +#[cfg(any(feature = "printing", feature = "full"))] +use std::ops::ControlFlow; + +#[cfg(feature = "full")] +pub(crate) fn requires_semi_to_be_stmt(expr: &Expr) -> bool { + match expr { + Expr::Macro(expr) => !expr.mac.delimiter.is_brace(), + _ => requires_comma_to_be_match_arm(expr), + } +} + +#[cfg(feature = "full")] +pub(crate) fn requires_comma_to_be_match_arm(expr: &Expr) -> bool { + match expr { + Expr::If(_) + | Expr::Match(_) + | Expr::Block(_) | Expr::Unsafe(_) // both under ExprKind::Block in rustc + | Expr::While(_) + | Expr::Loop(_) + | Expr::ForLoop(_) + | Expr::TryBlock(_) + | Expr::Const(_) => false, + + Expr::Array(_) + | Expr::Assign(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Binary(_) + | Expr::Break(_) + | Expr::Call(_) + | Expr::Cast(_) + | Expr::Closure(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Yield(_) + | Expr::Verbatim(_) => true, + } +} + +#[cfg(feature = "printing")] +pub(crate) fn trailing_unparameterized_path(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Path(t) => match last_type_in_path(&t.path) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Macro(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) + | Type::Verbatim(_) => return false, + } + } + + fn last_type_in_path(path: &Path) -> ControlFlow { + match &path.segments.last().unwrap().arguments { + PathArguments::None => ControlFlow::Break(true), + PathArguments::AngleBracketed(_) => ControlFlow::Break(false), + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => ControlFlow::Break(false), + ReturnType::Type(_, ret) => ControlFlow::Continue(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => last_type_in_path(&t.path), + TypeParamBound::Lifetime(_) + | TypeParamBound::PreciseCapture(_) + | TypeParamBound::Verbatim(_) => ControlFlow::Break(false), + } + } +} + +/// Whether the expression's first token is the label of a loop/block. +#[cfg(all(feature = "printing", feature = "full"))] +pub(crate) fn expr_leading_label(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Block(e) => return e.label.is_some(), + Expr::ForLoop(e) => return e.label.is_some(), + Expr::Loop(e) => return e.label.is_some(), + Expr::While(e) => return e.label.is_some(), + + Expr::Assign(e) => expr = &e.left, + Expr::Await(e) => expr = &e.base, + Expr::Binary(e) => expr = &e.left, + Expr::Call(e) => expr = &e.func, + Expr::Cast(e) => expr = &e.expr, + Expr::Field(e) => expr = &e.base, + Expr::Index(e) => expr = &e.expr, + Expr::MethodCall(e) => expr = &e.receiver, + Expr::Range(e) => match &e.start { + Some(start) => expr = start, + None => return false, + }, + Expr::Try(e) => expr = &e.expr, + + Expr::Array(_) + | Expr::Async(_) + | Expr::Break(_) + | Expr::Closure(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Unsafe(_) + | Expr::Verbatim(_) + | Expr::Yield(_) => return false, + } + } +} + +/// Whether the expression's last token is `}`. +#[cfg(feature = "full")] +pub(crate) fn expr_trailing_brace(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Async(_) + | Expr::Block(_) + | Expr::Const(_) + | Expr::ForLoop(_) + | Expr::If(_) + | Expr::Loop(_) + | Expr::Match(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Unsafe(_) + | Expr::While(_) => return true, + + Expr::Assign(e) => expr = &e.right, + Expr::Binary(e) => expr = &e.right, + Expr::Break(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Cast(e) => return type_trailing_brace(&e.ty), + Expr::Closure(e) => expr = &e.body, + Expr::Let(e) => expr = &e.expr, + Expr::Macro(e) => return e.mac.delimiter.is_brace(), + Expr::Range(e) => match &e.end { + Some(end) => expr = end, + None => return false, + }, + Expr::RawAddr(e) => expr = &e.expr, + Expr::Reference(e) => expr = &e.expr, + Expr::Return(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Unary(e) => expr = &e.expr, + Expr::Verbatim(e) => return tokens_trailing_brace(e), + Expr::Yield(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + + Expr::Array(_) + | Expr::Await(_) + | Expr::Call(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Repeat(_) + | Expr::Try(_) + | Expr::Tuple(_) => return false, + } + } + + fn type_trailing_brace(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Macro(t) => return t.mac.delimiter.is_brace(), + Type::Path(t) => match last_type_in_path(&t.path) { + Some(t) => ty = t, + None => return false, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Verbatim(t) => return tokens_trailing_brace(t), + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) => return false, + } + } + } + + fn last_type_in_path(path: &Path) -> Option<&Type> { + match &path.segments.last().unwrap().arguments { + PathArguments::None | PathArguments::AngleBracketed(_) => None, + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => None, + ReturnType::Type(_, ret) => Some(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => match last_type_in_path(&t.path) { + Some(t) => ControlFlow::Continue(t), + None => ControlFlow::Break(false), + }, + TypeParamBound::Lifetime(_) | TypeParamBound::PreciseCapture(_) => { + ControlFlow::Break(false) + } + TypeParamBound::Verbatim(t) => ControlFlow::Break(tokens_trailing_brace(t)), + } + } + + fn tokens_trailing_brace(tokens: &TokenStream) -> bool { + if let Some(TokenTree::Group(last)) = tokens.clone().into_iter().last() { + last.delimiter() == Delimiter::Brace + } else { + false + } + } +} diff --git a/rust/syn/custom_keyword.rs b/rust/syn/custom_keyword.rs new file mode 100644 index 000000000..ff2c415f1 --- /dev/null +++ b/rust/syn/custom_keyword.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` or `keyword` and that the resulting parser be invoked +/// with a `kw::` or `keyword::` prefix. +/// +/// ``` +/// mod kw { +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// use syn::{LitBool, LitStr, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// mod kw { +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! custom_keyword { + ($ident:ident) => { + #[allow(non_camel_case_types)] + pub struct $ident { + #[allow(dead_code)] + pub span: $crate::__private::Span, + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( + span: __S, + ) -> $ident { + $ident { + span: $crate::__private::IntoSpans::into_spans(span), + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), + } + } + } + + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => { + // For peek. + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + if let $crate::__private::Some((ident, _rest)) = cursor.ident() { + ident == $crate::__private::stringify!($ident) + } else { + false + } + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let $crate::__private::Some((ident, rest)) = cursor.ident() { + if ident == $crate::__private::stringify!($ident) { + return $crate::__private::Ok(($ident { span: ident.span() }, rest)); + } + } + $crate::__private::Err(cursor.error($crate::__private::concat!( + "expected `", + $crate::__private::stringify!($ident), + "`", + ))) + }) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); + $crate::__private::TokenStreamExt::append(tokens, ident); + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str( + f, + $crate::__private::concat!( + "Keyword [", + $crate::__private::stringify!($ident), + "]", + ), + ) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; +} diff --git a/rust/syn/custom_punctuation.rs b/rust/syn/custom_punctuation.rs new file mode 100644 index 000000000..d37c0e9c3 --- /dev/null +++ b/rust/syn/custom_punctuation.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/// Define a type that supports parsing and printing a multi-character symbol +/// as if it were a punctuation token. +/// +/// # Usage +/// +/// ``` +/// syn::custom_punctuation!(LeftRightArrow, <=>); +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in punctuation token. +/// +/// - [Peeking] — `input.peek(LeftRightArrow)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #lrarrow ... )` +/// +/// - Construction from a [`Span`] — `let lrarrow = LeftRightArrow(sp)` +/// +/// - Construction from multiple [`Span`] — `let lrarrow = LeftRightArrow([sp, sp, sp])` +/// +/// - Field access to its spans — `let spans = lrarrow.spans` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// ``` +/// use proc_macro2::{TokenStream, TokenTree}; +/// use syn::parse::{Parse, ParseStream, Peek, Result}; +/// use syn::punctuated::Punctuated; +/// use syn::Expr; +/// +/// syn::custom_punctuation!(PathSeparator, ); +/// +/// // expr expr expr ... +/// struct PathSegments { +/// segments: Punctuated, +/// } +/// +/// impl Parse for PathSegments { +/// fn parse(input: ParseStream) -> Result { +/// let mut segments = Punctuated::new(); +/// +/// let first = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(first)?); +/// +/// while input.peek(PathSeparator) { +/// segments.push_punct(input.parse()?); +/// +/// let next = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(next)?); +/// } +/// +/// Ok(PathSegments { segments }) +/// } +/// } +/// +/// fn parse_until(input: ParseStream, end: E) -> Result { +/// let mut tokens = TokenStream::new(); +/// while !input.is_empty() && !input.peek(end) { +/// let next: TokenTree = input.parse()?; +/// tokens.extend(Some(next)); +/// } +/// Ok(tokens) +/// } +/// +/// fn main() { +/// let input = r#" a::b c::d::e "#; +/// let _: PathSegments = syn::parse_str(input).unwrap(); +/// } +/// ``` +#[macro_export] +macro_rules! custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + pub struct $ident { + #[allow(dead_code)] + pub spans: $crate::custom_punctuation_repr!($($tt)+), + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::custom_punctuation_repr!($($tt)+)>>( + spans: __S, + ) -> $ident { + let _validate_len = 0 $(+ $crate::custom_punctuation_len!(strict, $tt))*; + $ident { + spans: $crate::__private::IntoSpans::into_spans(spans) + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) + } + } + + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + let spans: $crate::custom_punctuation_repr!($($tt)+) = + $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; + Ok($ident(spans)) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_repr { + ($($tt:tt)+) => { + [$crate::__private::Span; 0 $(+ $crate::custom_punctuation_len!(lenient, $tt))+] + }; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +#[rustfmt::skip] +macro_rules! custom_punctuation_len { + ($mode:ident, &) => { 1 }; + ($mode:ident, &&) => { 2 }; + ($mode:ident, &=) => { 2 }; + ($mode:ident, @) => { 1 }; + ($mode:ident, ^) => { 1 }; + ($mode:ident, ^=) => { 2 }; + ($mode:ident, :) => { 1 }; + ($mode:ident, ,) => { 1 }; + ($mode:ident, $) => { 1 }; + ($mode:ident, .) => { 1 }; + ($mode:ident, ..) => { 2 }; + ($mode:ident, ...) => { 3 }; + ($mode:ident, ..=) => { 3 }; + ($mode:ident, =) => { 1 }; + ($mode:ident, ==) => { 2 }; + ($mode:ident, =>) => { 2 }; + ($mode:ident, >=) => { 2 }; + ($mode:ident, >) => { 1 }; + ($mode:ident, <-) => { 2 }; + ($mode:ident, <=) => { 2 }; + ($mode:ident, <) => { 1 }; + ($mode:ident, -) => { 1 }; + ($mode:ident, -=) => { 2 }; + ($mode:ident, !=) => { 2 }; + ($mode:ident, !) => { 1 }; + ($mode:ident, |) => { 1 }; + ($mode:ident, |=) => { 2 }; + ($mode:ident, ||) => { 2 }; + ($mode:ident, ::) => { 2 }; + ($mode:ident, %) => { 1 }; + ($mode:ident, %=) => { 2 }; + ($mode:ident, +) => { 1 }; + ($mode:ident, +=) => { 2 }; + ($mode:ident, #) => { 1 }; + ($mode:ident, ?) => { 1 }; + ($mode:ident, ->) => { 2 }; + ($mode:ident, ;) => { 1 }; + ($mode:ident, <<) => { 2 }; + ($mode:ident, <<=) => { 3 }; + ($mode:ident, >>) => { 2 }; + ($mode:ident, >>=) => { 3 }; + ($mode:ident, /) => { 1 }; + ($mode:ident, /=) => { 2 }; + ($mode:ident, *) => { 1 }; + ($mode:ident, *=) => { 2 }; + ($mode:ident, ~) => { 1 }; + (lenient, $tt:tt) => { 0 }; + (strict, $tt:tt) => {{ $crate::custom_punctuation_unexpected!($tt); 0 }}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_unexpected { + () => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! stringify_punct { + ($($tt:tt)+) => { + $crate::__private::concat!($($crate::__private::stringify!($tt)),+) + }; +} diff --git a/rust/syn/data.rs b/rust/syn/data.rs new file mode 100644 index 000000000..c32c1366c --- /dev/null +++ b/rust/syn/data.rs @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +use crate::expr::{Expr, Index, Member}; +use crate::ident::Ident; +use crate::punctuated::{self, Punctuated}; +use crate::restriction::{FieldMutability, Visibility}; +use crate::token; +use crate::ty::Type; + +ast_struct! { + /// An enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Variant { + pub attrs: Vec, + + /// Name of the variant. + pub ident: Ident, + + /// Content stored in the variant. + pub fields: Fields, + + /// Explicit discriminant: `Variant = 1` + pub discriminant: Option<(Token![=], Expr)>, + } +} + +ast_enum_of_structs! { + /// Data stored within an enum variant or struct. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Fields { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + Named(FieldsNamed), + + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + Unnamed(FieldsUnnamed), + + /// Unit struct or unit variant such as `None`. + Unit, + } +} + +ast_struct! { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsNamed { + pub brace_token: token::Brace, + pub named: Punctuated, + } +} + +ast_struct! { + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsUnnamed { + pub paren_token: token::Paren, + pub unnamed: Punctuated, + } +} + +impl Fields { + /// Get an iterator over the borrowed [`Field`] items in this object. This + /// iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + pub fn iter(&self) -> punctuated::Iter { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter(), + Fields::Named(f) => f.named.iter(), + Fields::Unnamed(f) => f.unnamed.iter(), + } + } + + /// Get an iterator over the mutably borrowed [`Field`] items in this + /// object. This iterator can be used to iterate over a named or unnamed + /// struct or variant's fields uniformly. + pub fn iter_mut(&mut self) -> punctuated::IterMut { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter_mut(), + Fields::Named(f) => f.named.iter_mut(), + Fields::Unnamed(f) => f.unnamed.iter_mut(), + } + } + + /// Returns the number of fields. + pub fn len(&self) -> usize { + match self { + Fields::Unit => 0, + Fields::Named(f) => f.named.len(), + Fields::Unnamed(f) => f.unnamed.len(), + } + } + + /// Returns `true` if there are zero fields. + pub fn is_empty(&self) -> bool { + match self { + Fields::Unit => true, + Fields::Named(f) => f.named.is_empty(), + Fields::Unnamed(f) => f.unnamed.is_empty(), + } + } + + return_impl_trait! { + /// Get an iterator over the fields of a struct or variant as [`Member`]s. + /// This iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + /// + /// # Example + /// + /// The following is a simplistic [`Clone`] derive for structs. (A more + /// complete implementation would additionally want to infer trait bounds on + /// the generic type parameters.) + /// + /// ``` + /// # use quote::quote; + /// # + /// fn derive_clone(input: &syn::ItemStruct) -> proc_macro2::TokenStream { + /// let ident = &input.ident; + /// let members = input.fields.members(); + /// let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + /// quote! { + /// impl #impl_generics Clone for #ident #ty_generics #where_clause { + /// fn clone(&self) -> Self { + /// Self { + /// #(#members: self.#members.clone()),* + /// } + /// } + /// } + /// } + /// } + /// ``` + /// + /// For structs with named fields, it produces an expression like `Self { a: + /// self.a.clone() }`. For structs with unnamed fields, `Self { 0: + /// self.0.clone() }`. And for unit structs, `Self {}`. + pub fn members(&self) -> impl Iterator + Clone + '_ [Members] { + Members { + fields: self.iter(), + index: 0, + } + } + } +} + +impl IntoIterator for Fields { + type Item = Field; + type IntoIter = punctuated::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + match self { + Fields::Unit => Punctuated::::new().into_iter(), + Fields::Named(f) => f.named.into_iter(), + Fields::Unnamed(f) => f.unnamed.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Fields { + type Item = &'a Field; + type IntoIter = punctuated::Iter<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut Fields { + type Item = &'a mut Field; + type IntoIter = punctuated::IterMut<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +ast_struct! { + /// A field of a struct or enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Field { + pub attrs: Vec, + + pub vis: Visibility, + + pub mutability: FieldMutability, + + /// Name of the field, if any. + /// + /// Fields of tuple structs have no names. + pub ident: Option, + + pub colon_token: Option, + + pub ty: Type, + } +} + +pub struct Members<'a> { + fields: punctuated::Iter<'a, Field>, + index: u32, +} + +impl<'a> Iterator for Members<'a> { + type Item = Member; + + fn next(&mut self) -> Option { + let field = self.fields.next()?; + let member = match &field.ident { + Some(ident) => Member::Named(ident.clone()), + None => { + #[cfg(all(feature = "parsing", feature = "printing"))] + let span = crate::spanned::Spanned::span(&field.ty); + #[cfg(not(all(feature = "parsing", feature = "printing")))] + let span = proc_macro2::Span::call_site(); + Member::Unnamed(Index { + index: self.index, + span, + }) + } + }; + self.index += 1; + Some(member) + } +} + +impl<'a> Clone for Members<'a> { + fn clone(&self) -> Self { + Members { + fields: self.fields.clone(), + index: self.index, + } + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Field, Fields, FieldsNamed, FieldsUnnamed, Variant}; + use crate::error::Result; + use crate::expr::Expr; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + #[cfg(not(feature = "full"))] + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::restriction::{FieldMutability, Visibility}; + #[cfg(not(feature = "full"))] + use crate::scan_expr::scan_expr; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Variant { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let _visibility: Visibility = input.parse()?; + let ident: Ident = input.parse()?; + let fields = if input.peek(token::Brace) { + Fields::Named(input.parse()?) + } else if input.peek(token::Paren) { + Fields::Unnamed(input.parse()?) + } else { + Fields::Unit + }; + let discriminant = if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + #[cfg(feature = "full")] + let discriminant: Expr = input.parse()?; + #[cfg(not(feature = "full"))] + let discriminant = { + let begin = input.fork(); + let ahead = input.fork(); + let mut discriminant: Result = ahead.parse(); + if discriminant.is_ok() { + input.advance_to(&ahead); + } else if scan_expr(input).is_ok() { + discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); + } + discriminant? + }; + Some((eq_token, discriminant)) + } else { + None + }; + Ok(Variant { + attrs, + ident, + fields, + discriminant, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsNamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsNamed { + brace_token: braced!(content in input), + named: content.parse_terminated(Field::parse_named, Token![,])?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsUnnamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsUnnamed { + paren_token: parenthesized!(content in input), + unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, + }) + } + } + + impl Field { + /// Parses a named (braced struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_named(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); + let ident = if unnamed_field { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let colon_token: Token![:] = input.parse()?; + + let ty: Type = if unnamed_field + && (input.peek(Token![struct]) + || input.peek(Token![union]) && input.peek2(token::Brace)) + { + let begin = input.fork(); + input.call(Ident::parse_any)?; + input.parse::()?; + Type::Verbatim(verbatim::between(&begin, input)) + } else { + input.parse()? + }; + + Ok(Field { + attrs, + vis, + mutability: FieldMutability::None, + ident: Some(ident), + colon_token: Some(colon_token), + ty, + }) + } + + /// Parses an unnamed (tuple struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_unnamed(input: ParseStream) -> Result { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + mutability: FieldMutability::None, + ident: None, + colon_token: None, + ty: input.parse()?, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::data::{Field, FieldsNamed, FieldsUnnamed, Variant}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Variant { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.ident.to_tokens(tokens); + self.fields.to_tokens(tokens); + if let Some((eq_token, disc)) = &self.discriminant { + eq_token.to_tokens(tokens); + disc.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsNamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.named.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsUnnamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.unnamed.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Field { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.vis.to_tokens(tokens); + if let Some(ident) = &self.ident { + ident.to_tokens(tokens); + TokensOrDefault(&self.colon_token).to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } +} diff --git a/rust/syn/derive.rs b/rust/syn/derive.rs new file mode 100644 index 000000000..27fd9f12c --- /dev/null +++ b/rust/syn/derive.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +use crate::data::{Fields, FieldsNamed, Variant}; +use crate::generics::Generics; +use crate::ident::Ident; +use crate::punctuated::Punctuated; +use crate::restriction::Visibility; +use crate::token; + +ast_struct! { + /// Data structure sent to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DeriveInput { + pub attrs: Vec, + pub vis: Visibility, + pub ident: Ident, + pub generics: Generics, + pub data: Data, + } +} + +ast_enum! { + /// The storage of a struct, enum or union data structure. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub enum Data { + Struct(DataStruct), + Enum(DataEnum), + Union(DataUnion), + } +} + +ast_struct! { + /// A struct input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataStruct { + pub struct_token: Token![struct], + pub fields: Fields, + pub semi_token: Option, + } +} + +ast_struct! { + /// An enum input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataEnum { + pub enum_token: Token![enum], + pub brace_token: token::Brace, + pub variants: Punctuated, + } +} + +ast_struct! { + /// An untagged union input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataUnion { + pub union_token: Token![union], + pub fields: FieldsNamed, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Fields, FieldsNamed, Variant}; + use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; + use crate::error::Result; + use crate::generics::{Generics, WhereClause}; + use crate::ident::Ident; + use crate::parse::{Parse, ParseStream}; + use crate::punctuated::Punctuated; + use crate::restriction::Visibility; + use crate::token; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for DeriveInput { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::()?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![struct]) { + let struct_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields, semi) = data_struct(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Struct(DataStruct { + struct_token, + fields, + semi_token: semi, + }), + }) + } else if lookahead.peek(Token![enum]) { + let enum_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, brace, variants) = data_enum(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Enum(DataEnum { + enum_token, + brace_token: brace, + variants, + }), + }) + } else if lookahead.peek(Token![union]) { + let union_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields) = data_union(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Union(DataUnion { + union_token, + fields, + }), + }) + } else { + Err(lookahead.error()) + } + } + } + + pub(crate) fn data_struct( + input: ParseStream, + ) -> Result<(Option, Fields, Option)> { + let mut lookahead = input.lookahead1(); + let mut where_clause = None; + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if where_clause.is_none() && lookahead.peek(token::Paren) { + let fields = input.parse()?; + + lookahead = input.lookahead1(); + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unnamed(fields), Some(semi))) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(token::Brace) { + let fields = input.parse()?; + Ok((where_clause, Fields::Named(fields), None)) + } else if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unit, Some(semi))) + } else { + Err(lookahead.error()) + } + } + + pub(crate) fn data_enum( + input: ParseStream, + ) -> Result<( + Option, + token::Brace, + Punctuated, + )> { + let where_clause = input.parse()?; + + let content; + let brace = braced!(content in input); + let variants = content.parse_terminated(Variant::parse, Token![,])?; + + Ok((where_clause, brace, variants)) + } + + pub(crate) fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + let where_clause = input.parse()?; + let fields = input.parse()?; + Ok((where_clause, fields)) + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::data::Fields; + use crate::derive::{Data, DeriveInput}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for DeriveInput { + fn to_tokens(&self, tokens: &mut TokenStream) { + for attr in self.attrs.outer() { + attr.to_tokens(tokens); + } + self.vis.to_tokens(tokens); + match &self.data { + Data::Struct(d) => d.struct_token.to_tokens(tokens), + Data::Enum(d) => d.enum_token.to_tokens(tokens), + Data::Union(d) => d.union_token.to_tokens(tokens), + } + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.data { + Data::Struct(data) => match &data.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + }, + Data::Enum(data) => { + self.generics.where_clause.to_tokens(tokens); + data.brace_token.surround(tokens, |tokens| { + data.variants.to_tokens(tokens); + }); + } + Data::Union(data) => { + self.generics.where_clause.to_tokens(tokens); + data.fields.to_tokens(tokens); + } + } + } + } +} diff --git a/rust/syn/discouraged.rs b/rust/syn/discouraged.rs new file mode 100644 index 000000000..2acd631ec --- /dev/null +++ b/rust/syn/discouraged.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Extensions to the parsing API with niche applicability. + +use crate::buffer::Cursor; +use crate::error::Result; +use crate::parse::{inner_unexpected, ParseBuffer, Unexpected}; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Delimiter; +use std::cell::Cell; +use std::mem; +use std::rc::Rc; + +/// Extensions to the `ParseStream` API to support speculative parsing. +pub trait Speculative { + /// Advance this parse stream to the position of a forked parse stream. + /// + /// This is the opposite operation to [`ParseStream::fork`]. You can fork a + /// parse stream, perform some speculative parsing, then join the original + /// stream to the fork to "commit" the parsing from the fork to the main + /// stream. + /// + /// If you can avoid doing this, you should, as it limits the ability to + /// generate useful errors. That said, it is often the only way to parse + /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem + /// is that when the fork fails to parse an `A`, it's impossible to tell + /// whether that was because of a syntax error and the user meant to provide + /// an `A`, or that the `A`s are finished and it's time to start parsing + /// `B`s. Use with care. + /// + /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by + /// parsing `B*` and removing the leading members of `A` from the + /// repetition, bypassing the need to involve the downsides associated with + /// speculative parsing. + /// + /// [`ParseStream::fork`]: ParseBuffer::fork + /// + /// # Example + /// + /// There has been chatter about the possibility of making the colons in the + /// turbofish syntax like `path::to::` no longer required by accepting + /// `path::to` in expression position. Specifically, according to [RFC + /// 2544], [`PathSegment`] parsing should always try to consume a following + /// `<` token as the start of generic arguments, and reset to the `<` if + /// that fails (e.g. the token is acting as a less-than operator). + /// + /// This is the exact kind of parsing behavior which requires the "fork, + /// try, commit" behavior that [`ParseStream::fork`] discourages. With + /// `advance_to`, we can avoid having to parse the speculatively parsed + /// content a second time. + /// + /// This change in behavior can be implemented in syn by replacing just the + /// `Parse` implementation for `PathSegment`: + /// + /// ``` + /// # use syn::ext::IdentExt; + /// use syn::parse::discouraged::Speculative; + /// # use syn::parse::{Parse, ParseStream}; + /// # use syn::{Ident, PathArguments, Result, Token}; + /// + /// pub struct PathSegment { + /// pub ident: Ident, + /// pub arguments: PathArguments, + /// } + /// # + /// # impl From for PathSegment + /// # where + /// # T: Into, + /// # { + /// # fn from(ident: T) -> Self { + /// # PathSegment { + /// # ident: ident.into(), + /// # arguments: PathArguments::None, + /// # } + /// # } + /// # } + /// + /// impl Parse for PathSegment { + /// fn parse(input: ParseStream) -> Result { + /// if input.peek(Token![super]) + /// || input.peek(Token![self]) + /// || input.peek(Token![Self]) + /// || input.peek(Token![crate]) + /// { + /// let ident = input.call(Ident::parse_any)?; + /// return Ok(PathSegment::from(ident)); + /// } + /// + /// let ident = input.parse()?; + /// if input.peek(Token![::]) && input.peek3(Token![<]) { + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(input.parse()?), + /// }); + /// } + /// if input.peek(Token![<]) && !input.peek(Token![<=]) { + /// let fork = input.fork(); + /// if let Ok(arguments) = fork.parse() { + /// input.advance_to(&fork); + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(arguments), + /// }); + /// } + /// } + /// Ok(PathSegment::from(ident)) + /// } + /// } + /// + /// # syn::parse_str::("a").unwrap(); + /// ``` + /// + /// # Drawbacks + /// + /// The main drawback of this style of speculative parsing is in error + /// presentation. Even if the lookahead is the "correct" parse, the error + /// that is shown is that of the "fallback" parse. To use the same example + /// as the turbofish above, take the following unfinished "turbofish": + /// + /// ```text + /// let _ = f<&'a fn(), for<'a> serde::>(); + /// ``` + /// + /// If this is parsed as generic arguments, we can provide the error message + /// + /// ```text + /// error: expected identifier + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^ + /// ``` + /// + /// but if parsed using the above speculative parsing, it falls back to + /// assuming that the `<` is a less-than when it fails to parse the generic + /// arguments, and tries to interpret the `&'a` as the start of a labelled + /// loop, resulting in the much less helpful error + /// + /// ```text + /// error: expected `:` + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^^ + /// ``` + /// + /// This can be mitigated with various heuristics (two examples: show both + /// forks' parse errors, or show the one that consumed more tokens), but + /// when you can control the grammar, sticking to something that can be + /// parsed LL(3) and without the LL(*) speculative parsing this makes + /// possible, displaying reasonable errors becomes much more simple. + /// + /// [RFC 2544]: https://github.com/rust-lang/rfcs/pull/2544 + /// [`PathSegment`]: crate::PathSegment + /// + /// # Performance + /// + /// This method performs a cheap fixed amount of work that does not depend + /// on how far apart the two streams are positioned. + /// + /// # Panics + /// + /// The forked stream in the argument of `advance_to` must have been + /// obtained by forking `self`. Attempting to advance to any other stream + /// will cause a panic. + fn advance_to(&self, fork: &Self); +} + +impl<'a> Speculative for ParseBuffer<'a> { + fn advance_to(&self, fork: &Self) { + if !crate::buffer::same_scope(self.cursor(), fork.cursor()) { + panic!("fork was not derived from the advancing parse stream"); + } + + let (self_unexp, self_sp) = inner_unexpected(self); + let (fork_unexp, fork_sp) = inner_unexpected(fork); + if !Rc::ptr_eq(&self_unexp, &fork_unexp) { + match (fork_sp, self_sp) { + // Unexpected set on the fork, but not on `self`, copy it over. + (Some((span, delimiter)), None) => { + self_unexp.set(Unexpected::Some(span, delimiter)); + } + // Unexpected unset. Use chain to propagate errors from fork. + (None, None) => { + fork_unexp.set(Unexpected::Chain(self_unexp)); + + // Ensure toplevel 'unexpected' tokens from the fork don't + // propagate up the chain by replacing the root `unexpected` + // pointer, only 'unexpected' tokens from existing group + // parsers should propagate. + fork.unexpected + .set(Some(Rc::new(Cell::new(Unexpected::None)))); + } + // Unexpected has been set on `self`. No changes needed. + (_, Some(_)) => {} + } + } + + // See comment on `cell` in the struct definition. + self.cell + .set(unsafe { mem::transmute::>(fork.cursor()) }); + } +} + +/// Extensions to the `ParseStream` API to support manipulating invisible +/// delimiters the same as if they were visible. +pub trait AnyDelimiter { + /// Returns the delimiter, the span of the delimiter token, and the nested + /// contents for further parsing. + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; +} + +impl<'a> AnyDelimiter for ParseBuffer<'a> { + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { + self.step(|cursor| { + if let Some((content, delimiter, span, rest)) = cursor.any_group() { + let scope = span.close(); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(self); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((delimiter, span, content), rest)) + } else { + Err(cursor.error("expected any delimiter")) + } + }) + } +} diff --git a/rust/syn/drops.rs b/rust/syn/drops.rs new file mode 100644 index 000000000..2c4ae333d --- /dev/null +++ b/rust/syn/drops.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::iter; +use std::mem::ManuallyDrop; +use std::ops::{Deref, DerefMut}; +use std::option; +use std::slice; + +#[repr(transparent)] +pub(crate) struct NoDrop(ManuallyDrop); + +impl NoDrop { + pub(crate) fn new(value: T) -> Self + where + T: TrivialDrop, + { + NoDrop(ManuallyDrop::new(value)) + } +} + +impl Deref for NoDrop { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for NoDrop { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +pub(crate) trait TrivialDrop {} + +impl TrivialDrop for iter::Empty {} +impl TrivialDrop for slice::Iter<'_, T> {} +impl TrivialDrop for slice::IterMut<'_, T> {} +impl TrivialDrop for option::IntoIter<&T> {} +impl TrivialDrop for option::IntoIter<&mut T> {} + +#[test] +fn test_needs_drop() { + use std::mem::needs_drop; + + struct NeedsDrop; + + impl Drop for NeedsDrop { + fn drop(&mut self) {} + } + + assert!(needs_drop::()); + + // Test each of the types with a handwritten TrivialDrop impl above. + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); +} diff --git a/rust/syn/error.rs b/rust/syn/error.rs new file mode 100644 index 000000000..6fa0faf7f --- /dev/null +++ b/rust/syn/error.rs @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +use crate::thread::ThreadBound; +use proc_macro2::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; +#[cfg(feature = "printing")] +use quote::ToTokens; +use std::fmt::{self, Debug, Display}; +use std::slice; +use std::vec; + +/// The result of a Syn parser. +pub type Result = std::result::Result; + +/// Error returned when a Syn parser cannot parse the input tokens. +/// +/// # Error reporting in proc macros +/// +/// The correct way to report errors back to the compiler from a procedural +/// macro is by emitting an appropriately spanned invocation of +/// [`compile_error!`] in the generated code. This produces a better diagnostic +/// message than simply panicking the macro. +/// +/// [`compile_error!`]: std::compile_error! +/// +/// When parsing macro input, the [`parse_macro_input!`] macro handles the +/// conversion to `compile_error!` automatically. +/// +/// [`parse_macro_input!`]: crate::parse_macro_input! +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::parse::{Parse, ParseStream, Result}; +/// use syn::{parse_macro_input, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as MyAttrArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// +/// struct MyAttrArgs { +/// # _k: [(); { stringify! { +/// ... +/// # }; 0 }] +/// } +/// +/// impl Parse for MyAttrArgs { +/// fn parse(input: ParseStream) -> Result { +/// # stringify! { +/// ... +/// # }; +/// # unimplemented!() +/// } +/// } +/// ``` +/// +/// For errors that arise later than the initial parsing stage, the +/// [`.to_compile_error()`] or [`.into_compile_error()`] methods can be used to +/// perform an explicit conversion to `compile_error!`. +/// +/// [`.to_compile_error()`]: Error::to_compile_error +/// [`.into_compile_error()`]: Error::into_compile_error +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, DeriveInput}; +/// # +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_derive(MyDerive)] +/// # }; +/// pub fn my_derive(input: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(input as DeriveInput); +/// +/// // fn(DeriveInput) -> syn::Result +/// expand::my_derive(input) +/// .unwrap_or_else(syn::Error::into_compile_error) +/// .into() +/// } +/// # +/// # mod expand { +/// # use proc_macro2::TokenStream; +/// # use syn::{DeriveInput, Result}; +/// # +/// # pub fn my_derive(input: DeriveInput) -> Result { +/// # unimplemented!() +/// # } +/// # } +/// ``` +pub struct Error { + messages: Vec, +} + +struct ErrorMessage { + // Span is implemented as an index into a thread-local interner to keep the + // size small. It is not safe to access from a different thread. We want + // errors to be Send and Sync to play nicely with ecosystem crates for error + // handling, so pin the span we're given to its original thread and assume + // it is Span::call_site if accessed from any other thread. + span: ThreadBound, + message: String, +} + +// Cannot use std::ops::Range because that does not implement Copy, +// whereas ThreadBound requires a Copy impl as a way to ensure no Drop impls +// are involved. +struct SpanRange { + start: Span, + end: Span, +} + +#[cfg(test)] +struct _Test +where + Error: Send + Sync; + +impl Error { + /// Usually the [`ParseStream::error`] method will be used instead, which + /// automatically uses the correct span from the current position of the + /// parse stream. + /// + /// Use `Error::new` when the error needs to be triggered on some span other + /// than where the parse stream is currently positioned. + /// + /// [`ParseStream::error`]: crate::parse::ParseBuffer::error + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, LitStr, Result, Token}; + /// use syn::parse::ParseStream; + /// + /// // Parses input that looks like `name = "string"` where the key must be + /// // the identifier `name` and the value may be any string literal. + /// // Returns the string literal. + /// fn parse_name(input: ParseStream) -> Result { + /// let name_token: Ident = input.parse()?; + /// if name_token != "name" { + /// // Trigger an error not on the current position of the stream, + /// // but on the position of the unexpected identifier. + /// return Err(Error::new(name_token.span(), "expected `name`")); + /// } + /// input.parse::()?; + /// let s: LitStr = input.parse()?; + /// Ok(s) + /// } + /// ``` + pub fn new(span: Span, message: T) -> Self { + return new(span, message.to_string()); + + fn new(span: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { + start: span, + end: span, + }), + message, + }], + } + } + } + + /// Creates an error with the specified message spanning the given syntax + /// tree node. + /// + /// Unlike the `Error::new` constructor, this constructor takes an argument + /// `tokens` which is a syntax tree node. This allows the resulting `Error` + /// to attempt to span all tokens inside of `tokens`. While you would + /// typically be able to use the `Spanned` trait with the above `Error::new` + /// constructor, implementation limitations today mean that + /// `Error::new_spanned` may provide a higher-quality error message on + /// stable Rust. + /// + /// When in doubt it's recommended to stick to `Error::new` (or + /// `ParseStream::error`)! + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn new_spanned(tokens: T, message: U) -> Self { + return new_spanned(tokens.into_token_stream(), message.to_string()); + + fn new_spanned(tokens: TokenStream, message: String) -> Error { + let mut iter = tokens.into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } + } + + /// The source location of the error. + /// + /// Spans are not thread-safe so this function returns `Span::call_site()` + /// if called from a different thread than the one on which the `Error` was + /// originally created. + pub fn span(&self) -> Span { + let SpanRange { start, end } = match self.messages[0].span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + start.join(end).unwrap_or(start) + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// The [`parse_macro_input!`] macro provides a convenient way to invoke + /// this method correctly in a procedural macro. + /// + /// [`compile_error!`]: std::compile_error! + /// [`parse_macro_input!`]: crate::parse_macro_input! + pub fn to_compile_error(&self) -> TokenStream { + self.messages + .iter() + .map(ErrorMessage::to_compile_error) + .collect() + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// [`compile_error!`]: std::compile_error! + /// + /// # Example + /// + /// ``` + /// # extern crate proc_macro; + /// # + /// use proc_macro::TokenStream; + /// use syn::{parse_macro_input, DeriveInput, Error}; + /// + /// # const _: &str = stringify! { + /// #[proc_macro_derive(MyTrait)] + /// # }; + /// pub fn derive_my_trait(input: TokenStream) -> TokenStream { + /// let input = parse_macro_input!(input as DeriveInput); + /// my_trait::expand(input) + /// .unwrap_or_else(Error::into_compile_error) + /// .into() + /// } + /// + /// mod my_trait { + /// use proc_macro2::TokenStream; + /// use syn::{DeriveInput, Result}; + /// + /// pub(crate) fn expand(input: DeriveInput) -> Result { + /// /* ... */ + /// # unimplemented!() + /// } + /// } + /// ``` + pub fn into_compile_error(self) -> TokenStream { + self.to_compile_error() + } + + /// Add another error message to self such that when `to_compile_error()` is + /// called, both errors will be emitted together. + pub fn combine(&mut self, another: Error) { + self.messages.extend(another.messages); + } +} + +impl ErrorMessage { + fn to_compile_error(&self) -> TokenStream { + let (start, end) = match self.span.get() { + Some(range) => (range.start, range.end), + None => (Span::call_site(), Span::call_site()), + }; + + // ::core::compile_error!($message) + TokenStream::from_iter([ + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("core", start)), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("compile_error", start)), + TokenTree::Punct({ + let mut punct = Punct::new('!', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Group({ + let mut group = Group::new(Delimiter::Brace, { + TokenStream::from_iter([TokenTree::Literal({ + let mut string = Literal::string(&self.message); + string.set_span(end); + string + })]) + }); + group.set_span(end); + group + }), + ]) + } +} + +#[cfg(feature = "parsing")] +pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { + if cursor.eof() { + Error::new(scope, format!("unexpected end of input, {}", message)) + } else { + let span = crate::buffer::open_span_of_group(cursor); + Error::new(span, message) + } +} + +#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] +pub(crate) fn new2(start: Span, end: Span, message: T) -> Error { + return new2(start, end, message.to_string()); + + fn new2(start: Span, end: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } +} + +impl Debug for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.messages.len() == 1 { + formatter + .debug_tuple("Error") + .field(&self.messages[0]) + .finish() + } else { + formatter + .debug_tuple("Error") + .field(&self.messages) + .finish() + } + } +} + +impl Debug for ErrorMessage { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.message, formatter) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(&self.messages[0].message) + } +} + +impl Clone for Error { + fn clone(&self) -> Self { + Error { + messages: self.messages.clone(), + } + } +} + +impl Clone for ErrorMessage { + fn clone(&self) -> Self { + ErrorMessage { + span: self.span, + message: self.message.clone(), + } + } +} + +impl Clone for SpanRange { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SpanRange {} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: LexError) -> Self { + Error::new(err.span(), err) + } +} + +impl IntoIterator for Error { + type Item = Error; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + messages: self.messages.into_iter(), + } + } +} + +pub struct IntoIter { + messages: vec::IntoIter, +} + +impl Iterator for IntoIter { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?], + }) + } +} + +impl<'a> IntoIterator for &'a Error { + type Item = Error; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Iter { + messages: self.messages.iter(), + } + } +} + +pub struct Iter<'a> { + messages: slice::Iter<'a, ErrorMessage>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?.clone()], + }) + } +} + +impl Extend for Error { + fn extend>(&mut self, iter: T) { + for err in iter { + self.combine(err); + } + } +} diff --git a/rust/syn/export.rs b/rust/syn/export.rs new file mode 100644 index 000000000..05dad324b --- /dev/null +++ b/rust/syn/export.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[doc(hidden)] +pub use std::clone::Clone; +#[doc(hidden)] +pub use std::cmp::{Eq, PartialEq}; +#[doc(hidden)] +pub use std::concat; +#[doc(hidden)] +pub use std::default::Default; +#[doc(hidden)] +pub use std::fmt::Debug; +#[doc(hidden)] +pub use std::hash::{Hash, Hasher}; +#[doc(hidden)] +pub use std::marker::Copy; +#[doc(hidden)] +pub use std::option::Option::{None, Some}; +#[doc(hidden)] +pub use std::result::Result::{Err, Ok}; +#[doc(hidden)] +pub use std::stringify; + +#[doc(hidden)] +pub type Formatter<'a> = std::fmt::Formatter<'a>; +#[doc(hidden)] +pub type FmtResult = std::fmt::Result; + +#[doc(hidden)] +pub type bool = std::primitive::bool; +#[doc(hidden)] +pub type str = std::primitive::str; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote; + +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream2 = proc_macro2::TokenStream; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::group::{parse_braces, parse_brackets, parse_parens}; + +#[doc(hidden)] +pub use crate::span::IntoSpans; + +#[cfg(all(feature = "parsing", feature = "printing"))] +#[doc(hidden)] +pub use crate::parse_quote::parse as parse_quote; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::parsing::{peek_punct, punct as parse_punct}; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use crate::token::printing::punct as print_punct; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::private::CustomToken; + +#[cfg(feature = "proc-macro")] +#[doc(hidden)] +pub type TokenStream = proc_macro::TokenStream; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote::{ToTokens, TokenStreamExt}; + +#[doc(hidden)] +pub struct private(pub(crate) ()); diff --git a/rust/syn/expr.rs b/rust/syn/expr.rs new file mode 100644 index 000000000..deb918436 --- /dev/null +++ b/rust/syn/expr.rs @@ -0,0 +1,4175 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::attr::Attribute; +#[cfg(all(feature = "parsing", feature = "full"))] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::ext::IdentExt as _; +#[cfg(feature = "full")] +use crate::generics::BoundLifetimes; +use crate::ident::Ident; +#[cfg(any(feature = "parsing", feature = "full"))] +use crate::lifetime::Lifetime; +use crate::lit::Lit; +use crate::mac::Macro; +use crate::op::{BinOp, UnOp}; +#[cfg(feature = "parsing")] +use crate::parse::ParseStream; +#[cfg(feature = "full")] +use crate::pat::Pat; +use crate::path::{AngleBracketedGenericArguments, Path, QSelf}; +use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +use crate::stmt::Block; +use crate::token; +#[cfg(feature = "full")] +use crate::ty::ReturnType; +use crate::ty::Type; +use proc_macro2::{Span, TokenStream}; +#[cfg(feature = "printing")] +use quote::IdentFragment; +#[cfg(feature = "printing")] +use std::fmt::{self, Display}; +use std::hash::{Hash, Hasher}; +#[cfg(all(feature = "parsing", feature = "full"))] +use std::mem; + +ast_enum_of_structs! { + /// A Rust expression. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature, but most of the variants are not available unless "full" is enabled.* + /// + /// # Syntax tree enums + /// + /// This type is a syntax tree enum. In Syn this and other syntax tree enums + /// are designed to be traversed using the following rebinding idiom. + /// + /// ``` + /// # use syn::Expr; + /// # + /// # fn example(expr: Expr) { + /// # const IGNORE: &str = stringify! { + /// let expr: Expr = /* ... */; + /// # }; + /// match expr { + /// Expr::MethodCall(expr) => { + /// /* ... */ + /// } + /// Expr::Cast(expr) => { + /// /* ... */ + /// } + /// Expr::If(expr) => { + /// /* ... */ + /// } + /// + /// /* ... */ + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// We begin with a variable `expr` of type `Expr` that has no fields + /// (because it is an enum), and by matching on it and rebinding a variable + /// with the same name `expr` we effectively imbue our variable with all of + /// the data fields provided by the variant that it turned out to be. So for + /// example above if we ended up in the `MethodCall` case then we get to use + /// `expr.receiver`, `expr.args` etc; if we ended up in the `If` case we get + /// to use `expr.cond`, `expr.then_branch`, `expr.else_branch`. + /// + /// This approach avoids repeating the variant names twice on every line. + /// + /// ``` + /// # use syn::{Expr, ExprMethodCall}; + /// # + /// # fn example(expr: Expr) { + /// // Repetitive; recommend not doing this. + /// match expr { + /// Expr::MethodCall(ExprMethodCall { method, args, .. }) => { + /// # } + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// In general, the name to which a syntax tree enum variant is bound should + /// be a suitable name for the complete syntax tree enum type. + /// + /// ``` + /// # use syn::{Expr, ExprField}; + /// # + /// # fn example(discriminant: ExprField) { + /// // Binding is called `base` which is the name I would use if I were + /// // assigning `*discriminant.base` without an `if let`. + /// if let Expr::Tuple(base) = *discriminant.base { + /// # } + /// # } + /// ``` + /// + /// A sign that you may not be choosing the right variable names is if you + /// see names getting repeated in your code, like accessing + /// `receiver.receiver` or `pat.pat` or `cond.cond`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum Expr { + /// A slice literal expression: `[a, b, c, d]`. + Array(ExprArray), + + /// An assignment expression: `a = compute()`. + Assign(ExprAssign), + + /// An async block: `async { ... }`. + Async(ExprAsync), + + /// An await expression: `fut.await`. + Await(ExprAwait), + + /// A binary operation: `a + b`, `a += b`. + Binary(ExprBinary), + + /// A blocked scope: `{ ... }`. + Block(ExprBlock), + + /// A `break`, with an optional label to break and an optional + /// expression. + Break(ExprBreak), + + /// A function call expression: `invoke(a, b)`. + Call(ExprCall), + + /// A cast expression: `foo as f64`. + Cast(ExprCast), + + /// A closure expression: `|a, b| a + b`. + Closure(ExprClosure), + + /// A const block: `const { ... }`. + Const(ExprConst), + + /// A `continue`, with an optional label. + Continue(ExprContinue), + + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + Field(ExprField), + + /// A for loop: `for pat in expr { ... }`. + ForLoop(ExprForLoop), + + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + Group(ExprGroup), + + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + If(ExprIf), + + /// A square bracketed indexing expression: `vector[2]`. + Index(ExprIndex), + + /// The inferred value of a const generic argument, denoted `_`. + Infer(ExprInfer), + + /// A `let` guard: `let Some(x) = opt`. + Let(ExprLet), + + /// A literal in place of an expression: `1`, `"foo"`. + Lit(ExprLit), + + /// Conditionless loop: `loop { ... }`. + Loop(ExprLoop), + + /// A macro invocation expression: `format!("{}", q)`. + Macro(ExprMacro), + + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + Match(ExprMatch), + + /// A method call expression: `x.foo::(a, b)`. + MethodCall(ExprMethodCall), + + /// A parenthesized expression: `(a + b)`. + Paren(ExprParen), + + /// A path like `std::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + Path(ExprPath), + + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + Range(ExprRange), + + /// Address-of operation: `&raw const place` or `&raw mut place`. + RawAddr(ExprRawAddr), + + /// A referencing operation: `&a` or `&mut a`. + Reference(ExprReference), + + /// An array literal constructed from one repeated element: `[0u8; N]`. + Repeat(ExprRepeat), + + /// A `return`, with an optional value to be returned. + Return(ExprReturn), + + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + Struct(ExprStruct), + + /// A try-expression: `expr?`. + Try(ExprTry), + + /// A try block: `try { ... }`. + TryBlock(ExprTryBlock), + + /// A tuple expression: `(a, b, c, d)`. + Tuple(ExprTuple), + + /// A unary operation: `!x`, `*x`. + Unary(ExprUnary), + + /// An unsafe block: `unsafe { ... }`. + Unsafe(ExprUnsafe), + + /// Tokens in expression position not interpreted by Syn. + Verbatim(TokenStream), + + /// A while loop: `while expr { ... }`. + While(ExprWhile), + + /// A yield expression: `yield expr`. + Yield(ExprYield), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match expr { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Expr::Array(expr) => {...} + // Expr::Assign(expr) => {...} + // ... + // Expr::Yield(expr) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A slice literal expression: `[a, b, c, d]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprArray #full { + pub attrs: Vec, + pub bracket_token: token::Bracket, + pub elems: Punctuated, + } +} + +ast_struct! { + /// An assignment expression: `a = compute()`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAssign #full { + pub attrs: Vec, + pub left: Box, + pub eq_token: Token![=], + pub right: Box, + } +} + +ast_struct! { + /// An async block: `async { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAsync #full { + pub attrs: Vec, + pub async_token: Token![async], + pub capture: Option, + pub block: Block, + } +} + +ast_struct! { + /// An await expression: `fut.await`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAwait #full { + pub attrs: Vec, + pub base: Box, + pub dot_token: Token![.], + pub await_token: Token![await], + } +} + +ast_struct! { + /// A binary operation: `a + b`, `a += b`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprBinary { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + +ast_struct! { + /// A blocked scope: `{ ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprBlock #full { + pub attrs: Vec, + pub label: Option